diff --git a/cmd/climc/shell/compute/cloudaccounts.go b/cmd/climc/shell/compute/cloudaccounts.go index 450648081a..94fc47dad1 100644 --- a/cmd/climc/shell/compute/cloudaccounts.go +++ b/cmd/climc/shell/compute/cloudaccounts.go @@ -38,6 +38,7 @@ func init() { cmd.CreateWithKeyword("create-openstack", &options.SOpenStackCloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-huawei", &options.SHuaweiCloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-hcso", &options.SHCSOAccountCreateOptions{}) + cmd.CreateWithKeyword("create-hcs", &options.SHCSAccountCreateOptions{}) cmd.CreateWithKeyword("create-ucloud", &options.SUcloudCloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-zstack", &options.SZStackCloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-s3", &options.SS3CloudAccountCreateOptions{}) @@ -63,6 +64,7 @@ func init() { cmd.UpdateWithKeyword("update-openstack", &options.SOpenStackCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-huawei", &options.SHuaweiCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-hcso", &options.SHCSOAccountUpdateOptions{}) + cmd.UpdateWithKeyword("update-hcs", &options.SHCSOAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-ucloud", &options.SUcloudCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-zstack", &options.SZStackCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-s3", &options.SS3CloudAccountUpdateOptions{}) @@ -86,6 +88,7 @@ func init() { cmd.PerformWithKeyword("update-credential-huawei", "update-credential", &options.SHuaweiCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-apsara", "update-credential", &options.SApsaraCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-hcso", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{}) + cmd.PerformWithKeyword("update-credential-hcs", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-ucloud", "update-credential", &options.SUcloudCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-zstack", "update-credential", &options.SZStackCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-s3", "update-credential", &options.SS3CloudAccountUpdateCredentialOptions{}) diff --git a/cmd/climc/shell/misc/feature.go b/cmd/climc/shell/misc/feature.go index adc78866f1..0811d6d23c 100644 --- a/cmd/climc/shell/misc/feature.go +++ b/cmd/climc/shell/misc/feature.go @@ -47,6 +47,7 @@ func init() { "apsara", "cloudpods", "hcso", + "hcs", "nutanix", "s3", "ceph", diff --git a/cmd/hcscli/main.go b/cmd/hcscli/main.go new file mode 100644 index 0000000000..1694354e31 --- /dev/null +++ b/cmd/hcscli/main.go @@ -0,0 +1,157 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "net/http" + "net/url" + "os" + + "golang.org/x/net/http/httpproxy" + + "yunion.io/x/structarg" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcs" + _ "yunion.io/x/onecloud/pkg/multicloud/hcs/shell" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +type BaseOptions struct { + Debug bool `help:"debug mode"` + AuthUrl string `help:"Auth url" default:"$HCS_AUTH_URL"` + AccessKey string `help:"Access key" default:"$HCS_ACCESS_KEY" metavar:"HCS_ACCESS_KEY"` + Secret string `help:"Secret" default:"$HCS_SECRET" metavar:"HCS_SECRET"` + RegionId string `help:"RegionId" default:"$HCS_REGION" metavar:"HCS_REGION"` + ProjectId string `help:"RegionId" default:"$HCS_PROJECT_ID" metavar:"HCS_PROJECT_ID"` + SUBCOMMAND string `help:"hcscli subcommand" subcommand:"true"` +} + +func getSubcommandParser() (*structarg.ArgumentParser, error) { + parse, e := structarg.NewArgumentParserWithHelp(&BaseOptions{}, + "hcscli", + "Command-line interface to hcs API.", + `See "hcscli COMMAND --help" for help on a specific command.`) + + if e != nil { + return nil, e + } + + subcmd := parse.GetSubcommand() + if subcmd == nil { + return nil, fmt.Errorf("No subcommand argument.") + } + + for _, v := range shellutils.CommandTable { + _, e := subcmd.AddSubParserWithHelp(v.Options, v.Command, v.Desc, v.Callback) + if e != nil { + return nil, e + } + } + return parse, nil +} + +func showErrorAndExit(e error) { + fmt.Fprintf(os.Stderr, "%s", e) + fmt.Fprintln(os.Stderr) + os.Exit(1) +} + +func newClient(options *BaseOptions) (*hcs.SRegion, error) { + if len(options.AccessKey) == 0 { + return nil, fmt.Errorf("Missing accessKey") + } + + if len(options.Secret) == 0 { + return nil, fmt.Errorf("Missing secret") + } + + if len(options.AuthUrl) == 0 { + return nil, fmt.Errorf("Missing authUrl") + } + + if len(options.RegionId) == 0 { + return nil, fmt.Errorf("Missing regionId") + } + + cfg := &httpproxy.Config{ + HTTPProxy: os.Getenv("HTTP_PROXY"), + HTTPSProxy: os.Getenv("HTTPS_PROXY"), + NoProxy: os.Getenv("NO_PROXY"), + } + cfgProxyFunc := cfg.ProxyFunc() + proxyFunc := func(req *http.Request) (*url.URL, error) { + return cfgProxyFunc(req.URL) + } + + cli, err := hcs.NewHcsClient( + hcs.NewHcsConfig( + options.AccessKey, + options.Secret, + options.ProjectId, + options.AuthUrl, + ).Debug(options.Debug). + CloudproviderConfig( + cloudprovider.ProviderConfig{ + ProxyFunc: proxyFunc, + }, + ), + ) + if err != nil { + return nil, err + } + + return cli.GetRegion(options.RegionId) +} + +func main() { + parser, e := getSubcommandParser() + if e != nil { + showErrorAndExit(e) + } + e = parser.ParseArgs(os.Args[1:], false) + options := parser.Options().(*BaseOptions) + + if parser.IsHelpSet() { + fmt.Print(parser.HelpString()) + return + } + subcmd := parser.GetSubcommand() + subparser := subcmd.GetSubParser() + if e != nil || subparser == nil { + if subparser != nil { + fmt.Print(subparser.Usage()) + } else { + fmt.Print(parser.Usage()) + } + showErrorAndExit(e) + return + } + suboptions := subparser.Options() + if subparser.IsHelpSet() { + fmt.Print(subparser.HelpString()) + return + } + var region *hcs.SRegion + region, e = newClient(options) + if e != nil { + showErrorAndExit(e) + } + e = subcmd.Invoke(region, suboptions) + if e != nil { + showErrorAndExit(e) + } +} diff --git a/go.mod b/go.mod index 28e4f16328..339a88b72c 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( github.com/gosuri/uitable v0.0.0-20160404203958-36ee7e946282 github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead github.com/huaweicloud/huaweicloud-sdk-go v1.0.26 + github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible github.com/jaypipes/ghw v0.9.1 github.com/jdcloud-api/jdcloud-sdk-go v1.55.0 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c @@ -88,7 +89,7 @@ require ( golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 - golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220916014741-473347a5e6e3 + golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987 google.golang.org/grpc v1.38.0 google.golang.org/protobuf v1.27.1 gopkg.in/fatih/set.v0 v0.2.1 @@ -175,6 +176,7 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/google/btree v1.0.0 // indirect + github.com/google/go-cmp v0.5.7 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect diff --git a/go.sum b/go.sum index 066a4ad260..36503b3656 100644 --- a/go.sum +++ b/go.sum @@ -418,6 +418,8 @@ github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0 github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/huaweicloud/huaweicloud-sdk-go v1.0.26 h1:aWTl4Ng9lZUW1DupYyqYiqkTwuTKMmqUZBKLmX0iWxo= github.com/huaweicloud/huaweicloud-sdk-go v1.0.26/go.mod h1:YHXxw/bm7AohI4jTY8Z43JYw+DPCDLqjxoLTqmcyja4= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible h1:tANYIteuFrosKbRYUk1Yo/OGJjbt4x3OVg211Qc60M0= +github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible/go.mod h1:l7VUhRbTKCzdOacdT4oWCwATKyvZqUOlOqr0Ous3k4s= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= @@ -434,6 +436,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -514,6 +517,10 @@ github.com/mdlayher/arp v0.0.0-20190313224443-98a83c8a2717/go.mod h1:eOj1DDj3NAZ github.com/mdlayher/ethernet v0.0.0-20190313224307-5b5fc417d966/go.mod h1:5s5p/sMJ6sNsFl6uCh85lkFGV8kLuIYJCRJLavVJwvg= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7 h1:lez6TS6aAau+8wXUP3G9I3TGlmPFEq2CTxBaRqY6AGE= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= +github.com/mdlayher/genetlink v0.0.0-20191004171646-5cf585d3b847/go.mod h1:LNhNWFVJapYK8zEjVHUIle4gy+Oahfc3UtcaqZ8Dz98= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v0.0.0-20191004170026-3c8695cb0643/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v0.0.0-20191008140946-2a17fd90af51/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= github.com/mdlayher/raw v0.0.0-20190313224157-43dbcdd7739d/go.mod h1:r1fbeITl2xL/zLbVnNHFyOzQJTgr/3fpf1lJX/cjzR8= github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= github.com/mdlayher/raw v0.0.0-20190606144222-a54781e5f38f h1:Xjvm7UTnKTwrlhbs+8qA6I0v1iX77rY3QxftSgvOVRk= @@ -524,6 +531,7 @@ github.com/mholt/caddy v0.10.11 h1:s8X+R8DuBbrrMuUTcWSxlDe567B0s5EDmiDBKSYsioY= github.com/mholt/caddy v0.10.11/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY= github.com/miekg/dns v1.1.25 h1:dFwPR6SfLtrSwgDcIq2bcU/gVutB4sNApq2HBdqcakg= github.com/miekg/dns v1.1.25/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= @@ -739,7 +747,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -800,7 +810,10 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -851,6 +864,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -860,11 +874,15 @@ golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190830023255-19e00faab6ad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -975,8 +993,9 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220916014741-473347a5e6e3 h1:ARxNdT6I+00ZyY5yRT/ZECkQti4iGrMZX9dvG/ao/LY= -golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220916014741-473347a5e6e3/go.mod h1:yp4gl6zOlnDGOZeWeDfMwQcsdOIQnMdhuPx9mwwWBL4= +golang.zx2c4.com/wireguard v0.0.20190908/go.mod h1:LhfXh5z6bLC2lW2ve6BzYZFwnnsXK3OQjySR0Yh2dO8= +golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987 h1:26OAgqBTufVr8WKonCEhhjO1oKsYhHv0iM5Dg92G1TM= +golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987/go.mod h1:7hq1rEDsx7/FWl8IEEnfH2Xhs6M2MNnjUfN0PeI8Rm0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= diff --git a/pkg/apis/compute/cloudaccount_const.go b/pkg/apis/compute/cloudaccount_const.go index 647d430968..2c2f882959 100644 --- a/pkg/apis/compute/cloudaccount_const.go +++ b/pkg/apis/compute/cloudaccount_const.go @@ -46,6 +46,7 @@ const ( CLOUD_PROVIDER_AWS = "Aws" CLOUD_PROVIDER_HUAWEI = "Huawei" CLOUD_PROVIDER_HCSO = "HCSO" + CLOUD_PROVIDER_HCS = "HCS" CLOUD_PROVIDER_OPENSTACK = "OpenStack" CLOUD_PROVIDER_UCLOUD = "Ucloud" CLOUD_PROVIDER_ZSTACK = "ZStack" @@ -103,7 +104,8 @@ const ( var ( CLOUD_PROVIDER_VALID_STATUS = []string{CLOUD_PROVIDER_CONNECTED} CLOUD_PROVIDER_VALID_HEALTH_STATUS = []string{CLOUD_PROVIDER_HEALTH_NORMAL, CLOUD_PROVIDER_HEALTH_NO_PERMISSION} - PRIVATE_CLOUD_PROVIDERS = []string{CLOUD_PROVIDER_ZSTACK, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_APSARA, CLOUD_PROVIDER_HCSO, CLOUD_PROVIDER_INCLOUD_SPHERE, CLOUD_PROVIDER_PROXMOX, CLOUD_PROVIDER_REMOTEFILE} + PRIVATE_CLOUD_PROVIDERS = []string{CLOUD_PROVIDER_ZSTACK, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_APSARA, + CLOUD_PROVIDER_HCSO, CLOUD_PROVIDER_HCS, CLOUD_PROVIDER_INCLOUD_SPHERE, CLOUD_PROVIDER_PROXMOX, CLOUD_PROVIDER_REMOTEFILE} CLOUD_PROVIDERS = []string{ CLOUD_PROVIDER_ONECLOUD, @@ -115,6 +117,7 @@ var ( CLOUD_PROVIDER_AWS, CLOUD_PROVIDER_HUAWEI, CLOUD_PROVIDER_HCSO, + CLOUD_PROVIDER_HCS, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_UCLOUD, CLOUD_PROVIDER_ZSTACK, @@ -160,6 +163,9 @@ var ( CLOUD_PROVIDER_HCSO: { HOST_TYPE_HCSO, }, + CLOUD_PROVIDER_HCS: { + HOST_TYPE_HCS, + }, CLOUD_PROVIDER_OPENSTACK: { HOST_TYPE_OPENSTACK, }, diff --git a/pkg/apis/compute/elasticcache.go b/pkg/apis/compute/elasticcache.go index fdf1e7bc09..48f051f09e 100644 --- a/pkg/apis/compute/elasticcache.go +++ b/pkg/apis/compute/elasticcache.go @@ -179,17 +179,23 @@ type ElasticcacheCreateInput struct { ElasticcacheSecgroupsInput // 主可用区名称或Id - Zone string `json:"zone"` + ZoneId string `json:"zone_id"` - // 备可用区名称或Id列表 + // swagger: ignore + Zone string `json:"zone" yunion-deprecated-by:"zone_id"` + + // 备可用区名称或Id列表, split by: , // 默认副本与主可用区一致 // 支持此参数的云厂商: 腾讯云 // required: false - SlaveZones []string `json:"slave_zones"` + SlaveZones string `json:"slave_zones"` // Ip子网名称或Id,建议使用Id // required: true - Network string `json:"network"` + NetworkId string `json:"network_id"` + + // swagger: ignore + Network string `json:"network" yunion-deprecated-by:"network_id"` // 网络类型 // enum: vpc, cLassic @@ -209,13 +215,20 @@ type ElasticcacheCreateInput struct { // required: false InstanceType string `json:"instance_type"` + NodeType string `json:"node_type"` + + // swagger: ignore + MemorySizeMb int `json:"memory_size_mb" yunion-deprecated-by:"capacity_mb"` + // 初始密码 // required: false Password string `json:"password"` + ResetPassword bool `json:"reset_password"` + // 安全组名称或Id // default: default - Secgroup string `json:"secgroup"` + SecgroupIds []string `json:"secgroup_ids"` // 内网IP // 阿里云、华为云此参数可选，其它公有云该参数无效 @@ -249,4 +262,10 @@ type ElasticcacheCreateInput struct { // enum: 22:00:00, 02:00:00, 06:00:00, 10:00:00, 14:00:00, 18:00:00 // required: false MaintainStartTime string `json:"maintain_start_time"` + + LocalCategory string `json:"local_category"` + + CapacityMb int `json:"capacity_mb"` + + CloudregionId string `json:"cloudregion_id"` } diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index b5f83b446e..fa62538e59 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -174,6 +174,7 @@ const ( HYPERVISOR_AZURE = "azure" HYPERVISOR_AWS = "aws" HYPERVISOR_HUAWEI = "huawei" + HYPERVISOR_HCS = "hcs" HYPERVISOR_HCSO = "hcso" HYPERVISOR_OPENSTACK = "openstack" HYPERVISOR_UCLOUD = "ucloud" @@ -240,6 +241,7 @@ var HYPERVISORS = []string{ HYPERVISOR_QCLOUD, HYPERVISOR_HUAWEI, HYPERVISOR_HCSO, + HYPERVISOR_HCS, HYPERVISOR_OPENSTACK, HYPERVISOR_UCLOUD, HYPERVISOR_ZSTACK, @@ -280,6 +282,7 @@ var PRIVATE_CLOUD_HYPERVISORS = []string{ HYPERVISOR_APSARA, HYPERVISOR_CLOUDPODS, HYPERVISOR_HCSO, + HYPERVISOR_HCS, HYPERVISOR_NUTANIX, HYPERVISOR_BINGO_CLOUD, HYPERVISOR_INCLOUD_SPHERE, @@ -301,6 +304,7 @@ var HYPERVISOR_HOSTTYPE = map[string]string{ HYPERVISOR_QCLOUD: HOST_TYPE_QCLOUD, HYPERVISOR_HUAWEI: HOST_TYPE_HUAWEI, HYPERVISOR_HCSO: HOST_TYPE_HCSO, + HYPERVISOR_HCS: HOST_TYPE_HCS, HYPERVISOR_OPENSTACK: HOST_TYPE_OPENSTACK, HYPERVISOR_UCLOUD: HOST_TYPE_UCLOUD, HYPERVISOR_ZSTACK: HOST_TYPE_ZSTACK, @@ -328,6 +332,7 @@ var HOSTTYPE_HYPERVISOR = map[string]string{ HOST_TYPE_QCLOUD: HYPERVISOR_QCLOUD, HOST_TYPE_HUAWEI: HYPERVISOR_HUAWEI, HOST_TYPE_HCSO: HYPERVISOR_HCSO, + HOST_TYPE_HCS: HYPERVISOR_HCS, HOST_TYPE_OPENSTACK: HYPERVISOR_OPENSTACK, HOST_TYPE_UCLOUD: HYPERVISOR_UCLOUD, HOST_TYPE_ZSTACK: HYPERVISOR_ZSTACK, diff --git a/pkg/apis/compute/host_const.go b/pkg/apis/compute/host_const.go index 47287175fc..ac0d0ab4f2 100644 --- a/pkg/apis/compute/host_const.go +++ b/pkg/apis/compute/host_const.go @@ -30,6 +30,7 @@ const ( HOST_TYPE_AZURE = "azure" HOST_TYPE_HUAWEI = "huawei" HOST_TYPE_HCSO = "hcso" + HOST_TYPE_HCS = "hcs" HOST_TYPE_OPENSTACK = "openstack" HOST_TYPE_UCLOUD = "ucloud" HOST_TYPE_ZSTACK = "zstack" @@ -118,6 +119,7 @@ var HOST_TYPES = []string{ HOST_TYPE_QCLOUD, HOST_TYPE_HUAWEI, HOST_TYPE_HCSO, + HOST_TYPE_HCS, HOST_TYPE_OPENSTACK, HOST_TYPE_UCLOUD, HOST_TYPE_ZSTACK, diff --git a/pkg/apis/compute/network_const.go b/pkg/apis/compute/network_const.go index ab9afb817b..65de080825 100644 --- a/pkg/apis/compute/network_const.go +++ b/pkg/apis/compute/network_const.go @@ -64,6 +64,7 @@ var ( REGIONAL_NETWORK_PROVIDERS = []string{ CLOUD_PROVIDER_HUAWEI, CLOUD_PROVIDER_HCSO, + CLOUD_PROVIDER_HCS, CLOUD_PROVIDER_CTYUN, CLOUD_PROVIDER_UCLOUD, CLOUD_PROVIDER_GOOGLE, diff --git a/pkg/apis/const.go b/pkg/apis/const.go index 2e78792ee9..d1315dffcf 100644 --- a/pkg/apis/const.go +++ b/pkg/apis/const.go @@ -55,6 +55,9 @@ const ( SYS_CLOUD_TAG_PREFIX = "sys:" // Such tags have inherited and isolated properties CLASS_TAT_PREFIX = "cls:" + + SKU_STATUS_AVAILABLE = "available" + SKU_STATUS_SOLDOUT = "soldout" ) var ( diff --git a/pkg/cloudmon/providerdriver/hcs.go b/pkg/cloudmon/providerdriver/hcs.go new file mode 100644 index 0000000000..b244f8829f --- /dev/null +++ b/pkg/cloudmon/providerdriver/hcs.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package providerdriver + +import ( + api "yunion.io/x/onecloud/pkg/apis/compute" +) + +type HcsCollect struct { + SCollectByResourceIdDriver +} + +func (self *HcsCollect) GetProvider() string { + return api.CLOUD_PROVIDER_HCS +} + +func init() { + Register(&HcsCollect{}) +} + +func (self *HcsCollect) IsSupportMetrics() bool { + return true +} diff --git a/pkg/cloudprovider/elasticcache.go b/pkg/cloudprovider/elasticcache.go index 9dfd596811..3696c0ba3b 100644 --- a/pkg/cloudprovider/elasticcache.go +++ b/pkg/cloudprovider/elasticcache.go @@ -19,7 +19,6 @@ import "yunion.io/x/onecloud/pkg/util/billing" // https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423019.html // https://help.aliyun.com/document_detail/60873.html?spm=a2c4g.11174283.6.715.7412dce0qSYemb type SCloudElasticCacheInput struct { - RegionId string // 地域 InstanceType string // 实例规格 redis.master.small.default CapacityGB int64 // 缓存容量 华为云此项参数必选 InstanceName string // 实例名称 @@ -38,7 +37,7 @@ type SCloudElasticCacheInput struct { EipId string // 绑定弹性IP MaintainBegin string // 维护时间窗开始时间，格式为HH:mm:ss MaintainEnd string // 维护时间窗结束时间，格式为HH:mm:ss - BC *billing.SBillingCycle // 包年包月 + BillingCycle *billing.SBillingCycle // 包年包月 ProjectId string Tags map[string]string } diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 3a9f79e5eb..b678128583 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -150,6 +150,7 @@ type ICloudRegion interface { CreateIDBInstance(desc *SManagedDBInstanceCreateConfig) (ICloudDBInstance, error) GetIElasticcaches() ([]ICloudElasticcache, error) + GetIElasticcacheSkus() ([]ICloudElasticcacheSku, error) GetIElasticcacheById(id string) (ICloudElasticcache, error) CreateIElasticcaches(ec *SCloudElasticCacheInput) (ICloudElasticcache, error) @@ -1052,6 +1053,34 @@ type ICloudDBInstanceAccountPrivilege interface { GetDBName() string } +type ICloudElasticcacheSku interface { + GetName() string + GetGlobalId() string + GetZoneId() string + GetSlaveZoneId() string + GetEngineArch() string + GetLocalCategory() string + GetPrepaidStatus() string + GetPostpaidStatus() string + GetEngine() string + GetEngineVersion() string + GetCpuArch() string + GetStorageType() string + GetMemorySizeMb() int + GetPerformanceType() string + GetNodeType() string + GetDiskSizeGb() int + GetShardNum() int + GetMaxShardNum() int + GetReplicasNum() int + GetMaxReplicasNum() int + GetMaxClients() int + GetMaxConnections() int + GetMaxInBandwidthMb() int + GetMaxMemoryMb() int + GetQps() int +} + type ICloudElasticcache interface { IVirtualResource IBillingResource diff --git a/pkg/compute/guestdrivers/hcs.go b/pkg/compute/guestdrivers/hcs.go new file mode 100644 index 0000000000..0f1216029f --- /dev/null +++ b/pkg/compute/guestdrivers/hcs.go @@ -0,0 +1,207 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guestdrivers + +import ( + "context" + "database/sql" + "fmt" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" +) + +type SHCSGuestDriver struct { + SManagedVirtualizedGuestDriver +} + +func init() { + driver := SHCSGuestDriver{} + models.RegisterGuestDriver(&driver) +} + +func (self *SHCSGuestDriver) GetHypervisor() string { + return api.HYPERVISOR_HCS +} + +func (self *SHCSGuestDriver) GetProvider() string { + return api.CLOUD_PROVIDER_HCS +} + +func (self *SHCSGuestDriver) DoScheduleSKUFilter() bool { + return false +} + +func (self *SHCSGuestDriver) DoScheduleStorageFilter() bool { + return false +} + +func (self *SHCSGuestDriver) GetComputeQuotaKeys(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PRIVATE_CLOUD + keys.Provider = api.CLOUD_PROVIDER_HCS + keys.Brand = api.CLOUD_PROVIDER_HCS + keys.Hypervisor = api.HYPERVISOR_HCS + return keys +} + +func (self *SHCSGuestDriver) GetDefaultSysDiskBackend() string { + return api.STORAGE_HUAWEI_SAS +} + +func (self *SHCSGuestDriver) GetMinimalSysDiskSizeGb() int { + return 40 +} + +func (self *SHCSGuestDriver) GetStorageTypes() []string { + storages, _ := models.StorageManager.GetStorageTypesByHostType(api.HYPERVISOR_HOSTTYPE[self.GetHypervisor()]) + return storages +} + +func (self *SHCSGuestDriver) ChooseHostStorage(host *models.SHost, guest *models.SGuest, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) { + return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil +} + +func (self *SHCSGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SHCSGuestDriver) GetAttachDiskStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SHCSGuestDriver) GetRebuildRootStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SHCSGuestDriver) GetChangeConfigStatus(guest *models.SGuest) ([]string, error) { + return []string{api.VM_READY}, nil +} + +func (self *SHCSGuestDriver) GetDeployStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SHCSGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { + if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) { + return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status) + } + if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_HUAWEI_SATA, api.STORAGE_HUAWEI_SAS, api.STORAGE_HUAWEI_SSD}) { + return fmt.Errorf("Cannot resize disk with unsupported volumes type %s", storage.StorageType) + } + + return nil +} + +func (self *SHCSGuestDriver) GetGuestInitialStateAfterCreate() string { + return api.VM_RUNNING +} + +func (self *SHCSGuestDriver) GetGuestInitialStateAfterRebuild() string { + return api.VM_RUNNING +} + +func (self *SHCSGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability { + return cloudprovider.SInstanceCapability{ + Hypervisor: self.GetHypervisor(), + Provider: self.GetProvider(), + DefaultAccount: cloudprovider.SDefaultAccount{ + Linux: cloudprovider.SOsDefaultAccount{ + DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER, + }, + Windows: cloudprovider.SOsDefaultAccount{ + DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER, + }, + }, + Storages: cloudprovider.Storage{ + DataDisk: []cloudprovider.StorageInfo{ + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SSD, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true}, + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SATA, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true}, + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SAS, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true}, + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_GPSSD, MaxSizeGb: 32768, MinSizeGb: 10, StepSizeGb: 1, Resizable: true}, + }, + SysDisk: []cloudprovider.StorageInfo{ + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SSD, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false}, + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SATA, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false}, + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_SAS, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false}, + cloudprovider.StorageInfo{StorageType: api.STORAGE_HUAWEI_GPSSD, MaxSizeGb: 1024, MinSizeGb: 40, StepSizeGb: 1, Resizable: false}, + }, + }, + } +} + +func (self *SHCSGuestDriver) RemoteDeployGuestSyncHost(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, iVM cloudprovider.ICloudVM) (cloudprovider.ICloudHost, error) { + if hostId := iVM.GetIHostId(); len(hostId) > 0 { + nh, err := db.FetchByExternalIdAndManagerId(models.HostManager, hostId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + return q.Equals("manager_id", host.ManagerId) + }) + if err != nil { + log.Debugf("failed to found new hostId(%s) for ivm %s(%s) error: %v", hostId, guest.Name, guest.Id, err) + if errors.Cause(err) != sql.ErrNoRows { + return nil, errors.Wrap(err, "FetchByExternalIdAndManagerId") + } + + // HYPERVISOR_HCS VM被部署到一台全新的宿主机 + zone, err := host.GetZone() + if err != nil { + log.Warningf("host %s GetZone: %s", host.GetId(), err) + } else { + _host, err := models.HostManager.NewFromCloudHost(ctx, userCred, iVM.GetIHost(), host.GetCloudprovider(), zone) + if err != nil { + log.Warningf("NewFromCloudHost %s: %s", iVM.GetIHostId(), err) + } else { + host = _host + } + } + } else { + host = nh.(*models.SHost) + } + } + + if host.GetId() != guest.HostId { + guest.OnScheduleToHost(ctx, userCred, host.GetId()) + } + + return host.GetIHost(ctx) +} + +func (self *SHCSGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool { + months := bc.GetMonths() + if (months >= 1 && months <= 9) || (months == 12) || (months == 24) || (months == 36) { + return true + } + + return false +} + +func (self *SHCSGuestDriver) IsNeedInjectPasswordByCloudInit() bool { + return true +} + +func (self *SHCSGuestDriver) IsSupportSetAutoRenew() bool { + return false +} diff --git a/pkg/compute/hostdrivers/hcs.go b/pkg/compute/hostdrivers/hcs.go new file mode 100644 index 0000000000..ab449020db --- /dev/null +++ b/pkg/compute/hostdrivers/hcs.go @@ -0,0 +1,67 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostdrivers + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SHCSHostDriver struct { + SManagedVirtualizationHostDriver +} + +func init() { + driver := SHCSHostDriver{} + models.RegisterHostDriver(&driver) +} + +func (self *SHCSHostDriver) GetHostType() string { + return api.HOST_TYPE_HCS +} + +func (self *SHCSHostDriver) GetHypervisor() string { + return api.HYPERVISOR_HCS +} + +// 系统盘必须至少40G +func (self *SHCSHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb int) error { + if sizeGb < 1 || sizeGb > 65536 { + return fmt.Errorf("The %s disk size must be in the range of 1G ~ 65536GB", storage.StorageType) + } + return nil +} + +func (self *SHCSHostDriver) ValidateResetDisk(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, snapshot *models.SSnapshot, guests []models.SGuest, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + if len(guests) >= 1 { + if disk.DiskType == api.DISK_TYPE_SYS { + for _, g := range guests { + if g.Status != api.VM_READY { + return nil, httperrors.NewBadRequestError("Server %s must in status ready", g.GetName()) + } + } + } else { + return nil, httperrors.NewBadRequestError("Disk must be detached") + } + } + return data, nil +} diff --git a/pkg/compute/hostdrivers/managedvirtual.go b/pkg/compute/hostdrivers/managedvirtual.go index 3064a8cce2..33fac8709f 100644 --- a/pkg/compute/hostdrivers/managedvirtual.go +++ b/pkg/compute/hostdrivers/managedvirtual.go @@ -52,7 +52,7 @@ func (self *SManagedVirtualizationHostDriver) CheckAndSetCacheImage(ctx context. } providerName := storageCache.GetProviderName() - if utils.IsInStringArray(providerName, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_UCLOUD}) { + if utils.IsInStringArray(providerName, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS, api.CLOUD_PROVIDER_UCLOUD}) { image.OsVersion = input.OsFullVersion } diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index cffd551020..640fa3e4fd 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -1130,6 +1130,33 @@ func syncNATSkus(ctx context.Context, userCred mcclient.TokenCredential, syncRes } } +func syncCacheSkus(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { + skus, err := func() ([]cloudprovider.ICloudElasticcacheSku, error) { + defer syncResults.AddRequestCost(ElasticcacheSkuManager)() + return remoteRegion.GetIElasticcacheSkus() + }() + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotImplemented { + return + } + msg := fmt.Sprintf("GetIElasticcacheSkus for region %s failed %s", remoteRegion.GetName(), err) + log.Errorf(msg) + return + } + result := func() compare.SyncResult { + defer syncResults.AddSqlCost(ElasticcacheSkuManager)() + return localRegion.SyncPrivateCloudCacheSkus(ctx, userCred, skus) + }() + + syncResults.Add(NasSkuManager, result) + + msg := result.Result() + log.Infof("SyncRedisSkus for region %s result: %s", localRegion.Name, msg) + if result.IsError() { + return + } +} + func syncDBInstanceResource(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, localInstance *SDBInstance, remoteInstance cloudprovider.ICloudDBInstance) { err := syncDBInstanceNetwork(ctx, userCred, syncResults, localInstance, remoteInstance) if err != nil { @@ -1767,6 +1794,9 @@ func syncPublicCloudProviderInfo( if cloudprovider.IsSupportNAT(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NAT) { syncNATSkus(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) } + if cloudprovider.IsSupportElasticCache(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_CACHE) { + syncCacheSkus(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) + } } // no need to lock public cloud region as cloud region for public cloud is readonly diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 17bbbc6453..f3f44fe8e0 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -606,11 +606,13 @@ func (manager *SDiskManager) validateDiskOnStorage(diskConfig *api.DiskConfig, s } } + var guestdriver IGuestDriver = nil if host, _ := storage.GetMasterHost(); host != nil { //公有云磁盘大小检查。 if err := host.GetHostDriver().ValidateDiskSize(storage, diskConfig.SizeMb>>10); err != nil { return httperrors.NewInputParameterError("%v", err) } + guestdriver = GetDriver(api.HOSTTYPE_HYPERVISOR[host.HostType]) } hoststorages := HoststorageManager.Query().SubQuery() hoststorage := make([]SHoststorage, 0) @@ -620,8 +622,10 @@ func (manager *SDiskManager) validateDiskOnStorage(diskConfig *api.DiskConfig, s if len(hoststorage) == 0 { return httperrors.NewInputParameterError("Storage[%s] must attach to a host", storage.Name) } - if int64(diskConfig.SizeMb) > storage.GetFreeCapacity() && !storage.IsEmulated { - return httperrors.NewInputParameterError("Not enough free space") + if guestdriver == nil || guestdriver.DoScheduleStorageFilter() { + if int64(diskConfig.SizeMb) > storage.GetFreeCapacity() && !storage.IsEmulated { + return httperrors.NewInputParameterError("Not enough free space") + } } return nil } @@ -2191,7 +2195,7 @@ func (self *SDisk) PerformPurge(ctx context.Context, userCred mcclient.TokenCred } provider := self.GetCloudprovider() - if provider != nil && utils.IsInStringArray(provider.Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if provider != nil && utils.IsInStringArray(provider.Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { cnt, err := self.GetSnapshotCount() if err != nil { return nil, httperrors.NewInternalServerError("GetSnapshotCount fail %s", err) @@ -2206,7 +2210,7 @@ func (self *SDisk) PerformPurge(ctx context.Context, userCred mcclient.TokenCred func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { if !jsonutils.QueryBoolean(query, "delete_snapshots", false) { - if provider := self.GetCloudprovider(); provider != nil && utils.IsInStringArray(provider.Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if provider := self.GetCloudprovider(); provider != nil && utils.IsInStringArray(provider.Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { cnt, err := self.GetSnapshotCount() if err != nil { return httperrors.NewInternalServerError("GetSnapshotCount fail %s", err) diff --git a/pkg/compute/models/elasticcache_instances.go b/pkg/compute/models/elasticcache_instances.go index 66c2d3a1d5..3189baa588 100644 --- a/pkg/compute/models/elasticcache_instances.go +++ b/pkg/compute/models/elasticcache_instances.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -44,6 +45,7 @@ import ( "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/billing" bc "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/choices" "yunion.io/x/onecloud/pkg/util/logclient" @@ -122,9 +124,6 @@ type SElasticcache struct { // 带宽 Bandwidth int `nullable:"true" list:"user" create:"optional"` - // 安全组 - SecurityGroupId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" json:"security_group_id"` - // 内网DNS PrivateDNS string `width:"256" charset:"ascii" nullable:"true" list:"user" create:"optional" json:"private_dns"` @@ -779,48 +778,86 @@ func (manager *SElasticcacheManager) getElasticcachesByProviderId(providerId str return instances, nil } -func (manager *SElasticcacheManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { - _data := api.ElasticcacheCreateInput{} - err := data.Unmarshal(&_data) - if err != nil { - return nil, errors.Wrap(err, "ElasticcacheCreateInput.Unmarshal") - } - input, err := manager.validateCreateData(ctx, userCred, ownerId, query, _data) +func (manager *SElasticcacheManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { + var err error + input, err = manager.validateCreateData(ctx, userCred, ownerId, query, input) if err != nil { return nil, errors.Wrap(err, "validateCreateData") } - return input, nil } -func (manager *SElasticcacheManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) { +func (manager *SElasticcacheManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { return manager.validateCreateData(ctx, userCred, ownerId, query, input) } -func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) { - var region *SCloudregion - var provider *SCloudprovider - if len(input.Network) > 0 { - network, err := db.FetchByIdOrName(NetworkManager, userCred, strings.Split(input.Network, ",")[0]) +func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { + if len(input.NetworkId) == 0 { + return nil, httperrors.NewMissingParameterError("network_id") + } + networkObj, err := validators.ValidateModel(userCred, NetworkManager, &input.NetworkId) + if err != nil { + return nil, fmt.Errorf("getting network failed") + } + network := networkObj.(*SNetwork) + wire, err := network.GetWire() + if err != nil { + return nil, httperrors.NewGeneralError(errors.Wrapf(err, "GetWire")) + } + if len(wire.ZoneId) > 0 { + input.ZoneId = wire.ZoneId + } + _, err = validators.ValidateModel(userCred, ZoneManager, &input.ZoneId) + if err != nil { + return nil, err + } + vpc, err := network.GetVpc() + if err != nil { + return nil, httperrors.NewGeneralError(errors.Wrapf(err, "GetVpc")) + } + input.VpcId = vpc.Id + region, err := vpc.GetRegion() + if err != nil { + return nil, httperrors.NewGeneralError(errors.Wrapf(err, "GetRegion")) + } + input.CloudregionId = region.Id + provider := vpc.GetCloudprovider() + input.ManagerId = provider.Id + skuObj, err := validators.ValidateModel(userCred, ElasticcacheSkuManager, &input.InstanceType) + if err != nil { + return nil, err + } + sku := skuObj.(*SElasticcacheSku) + input.Engine = sku.Engine + input.EngineVersion = sku.EngineVersion + input.InstanceType = sku.InstanceSpec + input.NodeType = sku.NodeType + input.LocalCategory = sku.LocalCategory + + if len(input.PrivateIp) > 0 { + _, err = netutils.NewIPV4Addr(input.PrivateIp) if err != nil { - return nil, fmt.Errorf("getting network failed") + return nil, httperrors.NewInputParameterError("invalid private ip %s", input.PrivateIp) } - region, _ = network.(*SNetwork).GetRegion() - vpc, _ := network.(*SNetwork).GetVpc() - provider = vpc.GetCloudprovider() } - if region == nil { - return nil, fmt.Errorf("getting region failed") + if len(input.SecgroupIds) == 0 { + input.SecgroupIds = []string{api.SECGROUP_DEFAULT_ID} + } + + for i := range input.SecgroupIds { + _, err = validators.ValidateModel(userCred, SecurityGroupManager, &input.SecgroupIds[i]) + if err != nil { + return nil, err + } } // postpiad billing cycle if input.BillingType == billing_api.BILLING_TYPE_POSTPAID { - _cycle := input.Duration - if len(_cycle) > 0 { - cycle, err := bc.ParseBillingCycle(_cycle) + if len(input.Duration) > 0 { + cycle, err := bc.ParseBillingCycle(input.Duration) if err != nil { - return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", _cycle) + return nil, httperrors.NewInputParameterError("invalid duration %s", input.Duration) } tm := time.Time{} @@ -828,9 +865,24 @@ func (manager *SElasticcacheManager) validateCreateData(ctx context.Context, use // .Format("2006-01-02 15:04:05") input.ExpiredAt = cycle.EndAt(tm) } + } else if input.BillingType == billing_api.BILLING_TYPE_PREPAID { + cycle, err := billing.ParseBillingCycle(input.BillingCycle) + if err != nil { + return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", input.BillingCycle) + } + input.BillingCycle = cycle.String() + } + + // validate password + if len(input.Password) > 0 { + err := seclib2.ValidatePassword(input.Password) + if err != nil { + return nil, err + } + } else if input.ResetPassword { + input.Password = seclib2.RandomPassword2(12) } - var err error input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput) if err != nil { return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ValidateCreateData") @@ -861,335 +913,37 @@ func (self *SElasticcache) PostCreate(ctx context.Context, userCred mcclient.Tok log.Errorf("CancelPendingUsage error %s", err) } - password, _ := data.GetString("password") - if reset, _ := data.Bool("reset_password"); reset && len(password) == 0 { - password = seclib2.RandomPassword2(12) - } - - params := jsonutils.NewDict() - params.Set("password", jsonutils.NewString(password)) - secgroupIds, err := data.Get("secgroup_ids") - if err == nil { - params.Set("secgroup_ids", secgroupIds) - } - self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_DEPLOYING, "") - if err := self.StartElasticcacheCreateTask(ctx, userCred, params, ""); err != nil { - log.Errorf("Failed to create elastic cache error: %v", err) - } + input := &api.ElasticcacheCreateInput{} + data.Unmarshal(input) + self.StartElasticcacheCreateTask(ctx, userCred, input, "") } -func (self *SElasticcache) StartElasticcacheCreateTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { - task, err := taskman.TaskManager.NewTask(ctx, "ElasticcacheCreateTask", self, userCred, params, parentTaskId, "", nil) +func (self *SElasticcache) StartElasticcacheCreateTask(ctx context.Context, userCred mcclient.TokenCredential, input *api.ElasticcacheCreateInput, parentTaskId string) { + self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_DEPLOYING, "") + params := jsonutils.Marshal(input).(*jsonutils.JSONDict) + err := func() error { + task, err := taskman.TaskManager.NewTask(ctx, "ElasticcacheCreateTask", self, userCred, params, parentTaskId, "", nil) + if err != nil { + return errors.Wrapf(err, "NewTask") + } + return task.ScheduleRun(nil) + }() if err != nil { - return err + self.SetStatus(userCred, api.ELASTIC_CACHE_STATUS_CREATE_FAILED, err.Error()) } - task.ScheduleRun(nil) - return nil } func (self *SElasticcache) GetSlaveZones() ([]SZone, error) { + zones := []SZone{} if len(self.SlaveZones) > 0 { - zones := []SZone{} sz := strings.Split(self.SlaveZones, ",") err := ZoneManager.Query().In("id", sz).All(&zones) if err != nil { return nil, errors.Wrap(err, "GetZones") } - - zoneMap := map[string]SZone{} - for i := range zones { - zoneMap[zones[i].GetId()] = zones[i] - } - - ret := make([]SZone, len(sz)) - for i := range sz { - z, ok := zoneMap[sz[i]] - if !ok { - return nil, fmt.Errorf("zone %s is not found", sz[i]) - } - - ret[i] = z - } - - return ret, nil + return zones, nil } - - return []SZone{}, nil -} - -/*func (self *SElasticcache) GetIRegion() (cloudprovider.ICloudRegion, error) { - provider, err := self.GetDriver() - if err != nil { - return nil, fmt.Errorf("No cloudprovider for elastic cache %s: %s", self.Name, err) - } - region := self.GetRegion() - if region == nil { - return nil, fmt.Errorf("failed to find region for elastic cache %s", self.Name) - } - return provider.GetIRegionById(region.ExternalId) -}*/ - -func (self *SElasticcache) GetCreateAliyunElasticcacheParams(ctx context.Context, data *jsonutils.JSONDict) (*cloudprovider.SCloudElasticCacheInput, error) { - input := &cloudprovider.SCloudElasticCacheInput{} - iregion, err := self.GetIRegion(ctx) - if err != nil { - return nil, fmt.Errorf("elastic cache %s(%s) region not found", self.Name, self.Id) - } else { - input.RegionId = iregion.GetId() - } - - input.InstanceType = self.InstanceType - input.InstanceName = self.GetName() - - if password, _ := data.GetString("password"); len(password) > 0 { - input.Password = password - } - - input.Engine = strings.Title(self.Engine) - input.EngineVersion = self.EngineVersion - input.PrivateIpAddress = self.PrivateIpAddr - - zone, _ := self.GetZone() - if zone != nil { - izone, err := iregion.GetIZoneById(zone.ExternalId) - if err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateAliyunElasticcacheParams.Zone") - } - input.ZoneIds = []string{izone.GetId()} - } - - switch self.BillingType { - case billing_api.BILLING_TYPE_PREPAID: - input.ChargeType = "PrePaid" - billingCycle, err := bc.ParseBillingCycle(self.BillingCycle) - if err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateAliyunElasticcacheParams.BillingCycle") - } - billingCycle.AutoRenew = self.AutoRenew - input.BC = &billingCycle - default: - input.ChargeType = "PostPaid" - } - - // todo: fix me - if len(self.NodeType) > 0 { - switch self.NodeType { - case "single": - input.NodeType = "STAND_ALONE" - case "double": - input.NodeType = "MASTER_SLAVE" - default: - input.NodeType = "" - } - } - - switch self.NetworkType { - case api.LB_NETWORK_TYPE_CLASSIC: - input.NetworkType = "CLASSIC" - default: - input.NetworkType = "VPC" - } - - if ivpc, err := db.FetchById(VpcManager, self.VpcId); err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateAliyunElasticcacheParams.Vpc") - } else { - if ivpc != nil { - vpc := ivpc.(*SVpc) - input.VpcId = vpc.ExternalId - } - } - - if inetwork, err := db.FetchById(NetworkManager, self.NetworkId); err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateAliyunElasticcacheParams.Network") - } else { - if inetwork != nil { - network := inetwork.(*SNetwork) - input.NetworkId = network.ExternalId - } - } - input.Tags, _ = self.GetAllUserMetadata() - - return input, nil -} - -func (self *SElasticcache) GetCreateHuaweiElasticcacheParams(ctx context.Context, data *jsonutils.JSONDict) (*cloudprovider.SCloudElasticCacheInput, error) { - input := &cloudprovider.SCloudElasticCacheInput{} - iregion, err := self.GetIRegion(ctx) - if err != nil { - return nil, fmt.Errorf("elastic cache %s(%s) region not found", self.Name, self.Id) - } else { - input.RegionId = iregion.GetId() - } - - if self.CapacityMB > 0 { - input.CapacityGB = int64(self.CapacityMB / 1024) - } - - input.InstanceType = self.InstanceType - input.InstanceName = self.GetName() - - if password, _ := data.GetString("password"); len(password) > 0 { - input.Password = password - } - - switch self.Engine { - case "redis": - input.Engine = "Redis" - case "memcache": - input.Engine = "Memcached" - } - input.EngineVersion = self.EngineVersion - input.PrivateIpAddress = self.PrivateIpAddr - - zone, _ := self.GetZone() - if zone != nil { - izone, err := iregion.GetIZoneById(zone.ExternalId) - if err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Zone") - } - input.ZoneIds = []string{izone.GetId()} - } - - switch self.BillingType { - case billing_api.BILLING_TYPE_PREPAID: - input.ChargeType = "PrePaid" - billingCycle, err := bc.ParseBillingCycle(self.BillingCycle) - if err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.BillingCycle") - } - billingCycle.AutoRenew = self.AutoRenew - input.BC = &billingCycle - default: - input.ChargeType = "PostPaid" - } - - if len(self.NodeType) > 0 { - input.NodeType = self.NodeType - } - - switch self.NetworkType { - case api.LB_NETWORK_TYPE_CLASSIC: - input.NetworkType = "CLASSIC" - default: - input.NetworkType = "VPC" - } - - if ivpc, err := db.FetchById(VpcManager, self.VpcId); err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Vpc") - } else { - if ivpc != nil { - vpc := ivpc.(*SVpc) - input.VpcId = vpc.ExternalId - } - } - - if inetwork, err := db.FetchById(NetworkManager, self.NetworkId); err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Network") - } else { - if inetwork != nil { - network := inetwork.(*SNetwork) - input.NetworkId = network.ExternalId - } - } - - // fill security group here - if len(self.SecurityGroupId) > 0 { - region, _ := self.GetRegion() - sgCache, err := SecurityGroupCacheManager.GetSecgroupCache(context.Background(), nil, self.SecurityGroupId, self.VpcId, region.Id, self.GetCloudprovider().Id, "") - if err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.SecurityGroup") - } - - if sgCache == nil { - return nil, errors.Wrap(fmt.Errorf("cached security group not found"), "elasticcache.GetCreateHuaweiElasticcacheParams.SecurityGroup") - } - - input.SecurityGroupIds = []string{sgCache.GetExternalId()} - } - - if len(self.MaintainEndTime) > 0 { - input.MaintainBegin = self.MaintainStartTime - input.MaintainEnd = self.MaintainEndTime - } - return input, nil -} - -func (self *SElasticcache) GetCreateQCloudElasticcacheParams(ctx context.Context, data *jsonutils.JSONDict) (*cloudprovider.SCloudElasticCacheInput, error) { - input := &cloudprovider.SCloudElasticCacheInput{} - iregion, err := self.GetIRegion(ctx) - if err != nil { - return nil, fmt.Errorf("elastic cache %s(%s) region not found", self.Name, self.Id) - } else { - input.RegionId = iregion.GetId() - } - - input.InstanceType = self.InstanceType - input.InstanceName = self.GetName() - - if password, _ := data.GetString("password"); len(password) > 0 { - input.Password = password - } - - zone, _ := self.GetZone() - if zone != nil { - zones := []SZone{*zone} - // slave zones - sz, err := self.GetSlaveZones() - if err != nil { - return nil, errors.Wrap(err, "GetSlaveZones") - } - - zones = append(zones, sz...) - for i := range zones { - izone, err := iregion.GetIZoneById(zones[i].ExternalId) - if err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Zone") - } - input.ZoneIds = append(input.ZoneIds, izone.GetId()) - } - } - - switch self.BillingType { - case billing_api.BILLING_TYPE_PREPAID: - input.ChargeType = "PrePaid" - billingCycle, err := bc.ParseBillingCycle(self.BillingCycle) - if err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.BillingCycle") - } - billingCycle.AutoRenew = self.AutoRenew - input.BC = &billingCycle - default: - input.ChargeType = "PostPaid" - } - - switch self.NetworkType { - case api.LB_NETWORK_TYPE_CLASSIC: - input.NetworkType = "CLASSIC" - default: - input.NetworkType = "VPC" - } - - if ivpc, err := db.FetchById(VpcManager, self.VpcId); err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Vpc") - } else { - if ivpc != nil { - vpc := ivpc.(*SVpc) - input.VpcId = vpc.ExternalId - } - } - - if inetwork, err := db.FetchById(NetworkManager, self.NetworkId); err != nil { - return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.Network") - } else { - if inetwork != nil { - network := inetwork.(*SNetwork) - input.NetworkId = network.ExternalId - } - } - - input.Tags, _ = self.GetAllUserMetadata() - - return input, nil + return zones, nil } func (self *SElasticcache) PerformRestart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { @@ -1983,30 +1737,14 @@ func (self *SElasticcache) validateSecgroupInput(secgroups []string) error { return nil } -func CheckingSecgroupIds(ctx context.Context, userCred mcclient.TokenCredential, secgroupIds []string) ([]string, error) { - secgroupNames := []string{} - for _, secgroupId := range secgroupIds { - secgrp, err := SecurityGroupManager.FetchByIdOrName(userCred, secgroupId) - if err != nil { - if errors.Cause(err) == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2("secgroup", secgroupId) - } - return nil, httperrors.NewGeneralError(errors.Wrapf(err, "FetchByIdOrName(%s)", secgroupId)) - } - - err = SecurityGroupManager.ValidateName(secgrp.GetName()) - if err != nil { - return nil, httperrors.NewInputParameterError("The secgroup name %s does not meet the requirements, please change the name", secgrp.GetName()) - } - - secgroupNames = append(secgroupNames, secgrp.GetName()) - } - - return secgroupNames, nil -} - func (self *SElasticcache) checkingSecgroupIds(ctx context.Context, userCred mcclient.TokenCredential, secgroupIds []string) ([]string, error) { - return CheckingSecgroupIds(ctx, userCred, secgroupIds) + for i := range secgroupIds { + _, err := validators.ValidateModel(userCred, SecurityGroupManager, &secgroupIds[i]) + if err != nil { + return nil, err + } + } + return secgroupIds, nil } // 返回 本次更新后secgroup id的全集、本次更增的secgroup id, 本次删除的secgroup id, error diff --git a/pkg/compute/models/elasticcache_skus.go b/pkg/compute/models/elasticcache_skus.go index cf58ae418c..189a7f9229 100644 --- a/pkg/compute/models/elasticcache_skus.go +++ b/pkg/compute/models/elasticcache_skus.go @@ -25,12 +25,14 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/stringutils2" @@ -683,3 +685,94 @@ func (manager *SElasticcacheSkuManager) PerformSyncSkus(ctx context.Context, use func (manager *SElasticcacheSkuManager) GetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query api.SkuTaskQueryInput) (jsonutils.JSONObject, error) { return GetPropertySkusSyncTasks(ctx, userCred, query) } + +func (self *SCloudregion) SyncPrivateCloudCacheSkus(ctx context.Context, userCred mcclient.TokenCredential, iskus []cloudprovider.ICloudElasticcacheSku) compare.SyncResult { + lockman.LockRawObject(ctx, self.Id, ElasticcacheSkuManager.Keyword()) + defer lockman.ReleaseRawObject(ctx, self.Id, ElasticcacheSkuManager.Keyword()) + + result := compare.SyncResult{} + + dbSkus, err := self.GetElasticcacheSkus() + if err != nil { + result.Error(err) + return result + } + + removed := make([]SElasticcacheSku, 0) + commondb := make([]SElasticcacheSku, 0) + commonext := make([]cloudprovider.ICloudElasticcacheSku, 0) + added := make([]cloudprovider.ICloudElasticcacheSku, 0) + + err = compare.CompareSets(dbSkus, iskus, &removed, &commondb, &commonext, &added) + if err != nil { + result.Error(err) + return result + } + + for i := 0; i < len(removed); i += 1 { + err = removed[i].Delete(ctx, userCred) + if err != nil { + result.DeleteError(err) + continue + } + result.Delete() + } + for i := 0; i < len(added); i += 1 { + err = self.newFromCloudElasticcacheSku(ctx, userCred, added[i]) + if err != nil { + result.AddError(err) + } else { + result.Add() + } + } + return result +} + +func (self *SCloudregion) newFromCloudElasticcacheSku(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudElasticcacheSku) error { + sku := &SElasticcacheSku{} + sku.SetModelManager(ElasticcacheSkuManager, sku) + sku.Name = ext.GetName() + sku.InstanceSpec = ext.GetName() + sku.Status = apis.SKU_STATUS_AVAILABLE + sku.CloudregionId = self.Id + sku.ExternalId = ext.GetGlobalId() + sku.Provider = self.Provider + sku.EngineArch = ext.GetEngineArch() + sku.LocalCategory = ext.GetLocalCategory() + sku.PrepaidStatus = ext.GetPrepaidStatus() + sku.PostpaidStatus = ext.GetPostpaidStatus() + sku.Engine = ext.GetEngine() + sku.EngineVersion = ext.GetEngineVersion() + sku.CpuArch = ext.GetCpuArch() + sku.StorageType = ext.GetStorageType() + sku.MemorySizeMB = ext.GetMemorySizeMb() + sku.PerformanceType = ext.GetPerformanceType() + sku.NodeType = ext.GetNodeType() + sku.DiskSizeGB = ext.GetDiskSizeGb() + sku.ShardNum = ext.GetShardNum() + sku.MaxShardNum = ext.GetMaxShardNum() + sku.ReplicasNum = ext.GetReplicasNum() + sku.MaxReplicasNum = ext.GetMaxReplicasNum() + sku.MaxClients = ext.GetMaxClients() + sku.MaxConnections = ext.GetMaxConnections() + sku.MaxInBandwidthMb = ext.GetMaxInBandwidthMb() + sku.MaxMemoryMB = ext.GetMaxMemoryMb() + sku.QPS = ext.GetQps() + + zones, err := self.GetZones() + if err != nil { + return errors.Wrapf(err, "GetZones") + } + zoneId := ext.GetZoneId() + slaveZoneId := ext.GetSlaveZoneId() + for i := range zones { + if len(zoneId) > 0 && strings.HasSuffix(zones[i].ExternalId, zoneId) { + sku.ZoneId = zones[i].Id + } + if len(slaveZoneId) > 0 && strings.HasSuffix(zones[i].ExternalId, slaveZoneId) { + sku.SlaveZoneId = zones[i].Id + } + } + + return ElasticcacheSkuManager.TableSpec().Insert(ctx, sku) +} diff --git a/pkg/compute/models/loadbalancercachedacls.go b/pkg/compute/models/loadbalancercachedacls.go index 447e0978ae..405ed64987 100644 --- a/pkg/compute/models/loadbalancercachedacls.go +++ b/pkg/compute/models/loadbalancercachedacls.go @@ -126,7 +126,7 @@ func (man *SCachedLoadbalancerAclManager) ValidateCreateData(ctx context.Context } } - if utils.IsInStringArray(providerV.Model.(*SCloudprovider).Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if utils.IsInStringArray(providerV.Model.(*SCloudprovider).Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", ownerId) if err := listenerV.Validate(data); err != nil { return nil, err @@ -307,7 +307,7 @@ func (self *SCachedLoadbalancerAcl) syncRemoveCloudLoadbalanceAcl(ctx context.Co func (acl *SCachedLoadbalancerAcl) SyncWithCloudLoadbalancerAcl(ctx context.Context, userCred mcclient.TokenCredential, extAcl cloudprovider.ICloudLoadbalancerAcl, projectId mcclient.IIdentityProvider) error { diff, err := db.UpdateWithLock(ctx, acl, func() error { // todo: 华为云acl没有name字段应此不需要同步名称 - if !utils.IsInStringArray(acl.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if !utils.IsInStringArray(acl.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { acl.Name = extAcl.GetName() } else { ext_listener_id := extAcl.GetAclListenerID() @@ -343,7 +343,7 @@ func (man *SCachedLoadbalancerAclManager) GetOrCreateCachedAcl(ctx context.Conte defer lockman.ReleaseClass(ctx, man, ownerProjId) listenerId := "" - if utils.IsInStringArray(lblis.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if utils.IsInStringArray(lblis.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { listenerId = lblis.Id } if lblis.GetProviderName() == api.CLOUD_PROVIDER_OPENSTACK { diff --git a/pkg/compute/models/loadbalancerlistenerrules.go b/pkg/compute/models/loadbalancerlistenerrules.go index aa551bc851..b6f82a3a33 100644 --- a/pkg/compute/models/loadbalancerlistenerrules.go +++ b/pkg/compute/models/loadbalancerlistenerrules.go @@ -724,7 +724,7 @@ func (lbr *SLoadbalancerListenerRule) constructFieldsFromCloudListenerRule(userC } if groupId := extRule.GetBackendGroupId(); len(groupId) > 0 { - if utils.IsInStringArray(lbr.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if utils.IsInStringArray(lbr.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { group, err := db.FetchByExternalId(HuaweiCachedLbbgManager, groupId) if err != nil { if err == sql.ErrNoRows { @@ -769,7 +769,7 @@ func (lbr *SLoadbalancerListenerRule) updateCachedLoadbalancerBackendGroupAssoci } switch lbr.GetProviderName() { - case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO: + case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS: _group, err := db.FetchByExternalId(HuaweiCachedLbbgManager, exteralLbbgId) if err != nil { if err == sql.ErrNoRows { diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go index 75a38d7d90..33d196a2aa 100644 --- a/pkg/compute/models/loadbalancerlisteners.go +++ b/pkg/compute/models/loadbalancerlisteners.go @@ -386,7 +386,7 @@ func (man *SLoadbalancerListenerManager) ValidateAcl(aclStatusV *validators.Vali return httperrors.NewMissingParameterError("acl_type") } } else { - if !utils.IsInStringArray(providerName, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if !utils.IsInStringArray(providerName, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { data.Set("acl_id", jsonutils.NewString("")) data.Set("cached_acl_id", jsonutils.NewString("")) } @@ -1014,7 +1014,7 @@ func (lblis *SLoadbalancerListener) constructFieldsFromCloudListener(userCred mc switch lblis.ListenerType { case api.LB_LISTENER_TYPE_UDP: - if !utils.IsInStringArray(lblis.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if !utils.IsInStringArray(lblis.GetProviderName(), []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { lblis.HealthCheckExp = extListener.GetHealthCheckExp() lblis.HealthCheckReq = extListener.GetHealthCheckReq() } @@ -1057,7 +1057,7 @@ func (lblis *SLoadbalancerListener) constructFieldsFromCloudListener(userCred mc groupId := extListener.GetBackendGroupId() switch lblis.GetProviderName() { - case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO: + case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS: if len(groupId) > 0 { group, err := db.FetchByExternalIdAndManagerId(HuaweiCachedLbbgManager, groupId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Equals("manager_id", lb.ManagerId) @@ -1150,7 +1150,7 @@ func (lblis *SLoadbalancerListener) updateCachedLoadbalancerBackendGroupAssociat } switch lblis.GetProviderName() { - case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO: + case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS: _group, err := db.FetchByExternalIdAndManagerId(HuaweiCachedLbbgManager, exteralLbbgId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Equals("manager_id", managerId) }) diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index 06ad4c2cd6..76fa601779 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -391,6 +391,8 @@ func (man *SLoadbalancerManager) ValidateCreateData( } vpc := _vpc.(*SVpc) region, _ = vpc.GetRegion() + input.ManagerId = vpc.ManagerId + input.CloudproviderId = vpc.ManagerId } else if len(input.ZoneId) > 0 { var zone *SZone zone, input.ZoneResourceInput, err = ValidateZoneResourceInput(userCred, input.ZoneResourceInput) @@ -594,7 +596,7 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(ctx context.Context, iRegio params.EgressMbps = lb.EgressMbps } - if lb.AddressType == api.LB_ADDR_TYPE_INTRANET || utils.IsInStringArray(lb.SManagedResourceBase.GetProviderName(), []string{api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_AWS, api.CLOUD_PROVIDER_QCLOUD}) { + if lb.AddressType == api.LB_ADDR_TYPE_INTRANET || utils.IsInStringArray(lb.SManagedResourceBase.GetProviderName(), []string{api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCS, api.CLOUD_PROVIDER_AWS, api.CLOUD_PROVIDER_QCLOUD}) { vpc, err := lb.GetVpc() if err != nil { return nil, err @@ -606,7 +608,7 @@ func (lb *SLoadbalancer) GetCreateLoadbalancerParams(ctx context.Context, iRegio params.VpcID = iVpc.GetId() } - if lb.AddressType == api.LB_ADDR_TYPE_INTRANET || utils.IsInStringArray(lb.SManagedResourceBase.GetProviderName(), []string{api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_AWS}) { + if lb.AddressType == api.LB_ADDR_TYPE_INTRANET || utils.IsInStringArray(lb.SManagedResourceBase.GetProviderName(), []string{api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS, api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_AWS}) { networks, err := lb.GetNetworks() if err != nil { return nil, fmt.Errorf("failed to find network for lb %s: %s", lb.Name, err) diff --git a/pkg/compute/models/mount_targets.go b/pkg/compute/models/mount_targets.go index 7cc8bb1e82..1cb33145ee 100644 --- a/pkg/compute/models/mount_targets.go +++ b/pkg/compute/models/mount_targets.go @@ -342,7 +342,7 @@ func (self *SMountTarget) ValidateDeleteCondition(ctx context.Context, info json if err != nil { return httperrors.NewGeneralError(errors.Wrapf(err, "GetRegion")) } - if utils.IsInStringArray(region.Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO}) { + if utils.IsInStringArray(region.Provider, []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS}) { return httperrors.NewNotSupportedError("not allow to delete") } return self.SStatusStandaloneResourceBase.ValidateDeleteCondition(ctx, nil) diff --git a/pkg/compute/models/nat_skus.go b/pkg/compute/models/nat_skus.go index 877bfbe90d..951972224d 100644 --- a/pkg/compute/models/nat_skus.go +++ b/pkg/compute/models/nat_skus.go @@ -187,6 +187,16 @@ func (self *SCloudregion) GetNatSkus() ([]SNatSku, error) { return skus, nil } +func (self *SCloudregion) GetElasticcacheSkus() ([]SElasticcacheSku, error) { + skus := []SElasticcacheSku{} + q := ElasticcacheSkuManager.Query().Equals("cloudregion_id", self.Id) + err := db.FetchModelObjects(NatSkuManager, q, &skus) + if err != nil { + return nil, errors.Wrapf(err, "db.FetchModelObjects") + } + return skus, nil +} + func (self SNatSku) GetGlobalId() string { return self.ExternalId } diff --git a/pkg/compute/models/purge.go b/pkg/compute/models/purge.go index eca249a983..e0645a2cc3 100644 --- a/pkg/compute/models/purge.go +++ b/pkg/compute/models/purge.go @@ -557,7 +557,7 @@ func (lbbg *SLoadbalancerBackendGroup) purgeCachedlbbg(ctx context.Context, user switch lbbg.GetProviderName() { case api.CLOUD_PROVIDER_AWS: return lbbg.purgeAwsCachedlbbg(ctx, userCred) - case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO: + case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS: return lbbg.purgeHuaweiCachedlbbg(ctx, userCred) } diff --git a/pkg/compute/models/regiondrivers.go b/pkg/compute/models/regiondrivers.go index 19b710833f..5a3ef24446 100644 --- a/pkg/compute/models/regiondrivers.go +++ b/pkg/compute/models/regiondrivers.go @@ -226,7 +226,7 @@ type IElasticcacheDriver interface { AllowCreateElasticcacheBackup(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, elasticcache *SElasticcache) error AllowUpdateElasticcacheAuthMode(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, elasticcache *SElasticcache) error - ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) + ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) ValidateCreateElasticcacheAccountData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) ValidateCreateElasticcacheAclData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) ValidateCreateElasticcacheBackupData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index e0afea66ed..15915d143f 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -615,7 +615,7 @@ func (self *SSecurityGroupCache) StartSecurityGroupCacheDeleteTask(ctx context.C } func (manager *SSecurityGroupCacheManager) InitializeData() error { - providerIds := CloudproviderManager.Query("id").In("provider", []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_CTYUN, api.CLOUD_PROVIDER_QCLOUD}).SubQuery() + providerIds := CloudproviderManager.Query("id").In("provider", []string{api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS, api.CLOUD_PROVIDER_CTYUN, api.CLOUD_PROVIDER_QCLOUD}).SubQuery() deprecatedSecgroups := []SSecurityGroupCache{} q := manager.Query().In("manager_id", providerIds).NotEquals("vpc_id", api.NORMAL_VPC_ID) diff --git a/pkg/compute/regiondrivers/aliyun.go b/pkg/compute/regiondrivers/aliyun.go index 083d2b8843..30c6053809 100644 --- a/pkg/compute/regiondrivers/aliyun.go +++ b/pkg/compute/regiondrivers/aliyun.go @@ -23,7 +23,6 @@ import ( "unicode" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/secrules" @@ -1066,178 +1065,11 @@ func (self *SAliyunRegionDriver) ValidateDBInstanceAccountPrivilege(ctx context. return nil } -func (self *SAliyunRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) { - data := input.JSON(input) - zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId) - networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId) - - // billing cycle - billingTypeV := validators.NewStringChoicesValidator("billing_type", choices.NewChoices(billing_api.BILLING_TYPE_PREPAID, billing_api.BILLING_TYPE_POSTPAID)) - networkTypeV := validators.NewStringChoicesValidator("network_type", choices.NewChoices(api.LB_NETWORK_TYPE_VPC, api.LB_NETWORK_TYPE_CLASSIC)).Default(api.LB_NETWORK_TYPE_VPC).Optional(true) - engineV := validators.NewStringChoicesValidator("engine", choices.NewChoices("redis", "memcache")) - engineVersionV := validators.NewStringChoicesValidator("engine_version", choices.NewChoices("2.8", "4.0", "5.0")) - privateIpV := validators.NewIPv4AddrValidator("private_ip").Optional(true) - - keyV := map[string]validators.IValidator{ - "zone": zoneV, - "billing_type": billingTypeV, - "network_type": networkTypeV, - "network": networkV, - "engine": engineV, - "engine_version": engineVersionV, - "private_ip": privateIpV, +func (self *SAliyunRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { + if !utils.IsInStringArray(input.Engine, []string{"redis", "memcache"}) { + return nil, httperrors.NewInputParameterError("invalid engine %s", input.Engine) } - if err := RunValidators(keyV, data, false); err != nil { - return nil, err - } - - // validate password - if password, _ := data.GetString("password"); len(password) > 0 { - err := seclib2.ValidatePassword(password) - if err != nil { - return nil, err - } - } - - // validate sku - billingType, _ := data.GetString("billing_type") - zone := zoneV.Model.(*models.SZone) - network := networkV.Model.(*models.SNetwork) - if sku, err := data.GetString("instance_type"); err != nil || len(sku) == 0 { - return nil, httperrors.NewMissingParameterError("instance_type") - } else { - _skuModel, err := db.FetchByIdOrName(models.ElasticcacheSkuManager, userCred, sku) - if err != nil { - return nil, err - } - - skuModel := _skuModel.(*models.SElasticcacheSku) - if err := ValidateElasticcacheSku(zone.Id, billingType, skuModel, network); err != nil { - return nil, err - } else { - data.Set("instance_type", jsonutils.NewString(skuModel.InstanceSpec)) - data.Set("node_type", jsonutils.NewString(skuModel.NodeType)) - data.Set("local_category", jsonutils.NewString(skuModel.LocalCategory)) - } - } - - // billing cycle - if billingType == billing_api.BILLING_TYPE_PREPAID { - billingCycle, err := data.GetString("billing_cycle") - if err != nil { - return nil, httperrors.NewMissingParameterError("billing_cycle") - } - - cycle, err := billing.ParseBillingCycle(billingCycle) - if err != nil { - return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", billingCycle) - } - - data.Set("billing_cycle", jsonutils.NewString(cycle.String())) - } - - vpc, _ := network.GetVpc() - if vpc == nil { - return nil, httperrors.NewNotFoundError("network %s related vpc not found", network.GetId()) - } - data.Set("vpc_id", jsonutils.NewString(vpc.Id)) - data.Set("manager_id", jsonutils.NewString(vpc.ManagerId)) - data.Set("cloudregion_id", jsonutils.NewString(zone.GetCloudRegionId())) - return data, nil -} - -func (self *SAliyunRegionDriver) RequestCreateElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask, data *jsonutils.JSONDict) error { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - iRegion, err := ec.GetIRegion(ctx) - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.GetIRegion") - } - - iprovider, err := db.FetchById(models.CloudproviderManager, ec.GetCloudproviderId()) - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.GetProvider") - } - - params, err := ec.GetCreateAliyunElasticcacheParams(ctx, task.GetParams()) - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.GetCreateAliyunElasticcacheParams") - } - - provider := iprovider.(*models.SCloudprovider) - params.ProjectId, err = provider.SyncProject(ctx, userCred, ec.ProjectId) - if err != nil { - log.Errorf("failed to sync project %s for create %s elastic cache %s error: %v", ec.ProjectId, provider.Provider, ec.Name, err) - } - - iec, err := iRegion.CreateIElasticcaches(params) - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.CreateIElasticcaches") - } - - err = cloudprovider.WaitStatusWithDelay(iec, api.ELASTIC_CACHE_STATUS_RUNNING, 30*time.Second, 15*time.Second, 600*time.Second) - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.WaitStatusWithDelay") - } - - ec.SetModelManager(models.ElasticcacheManager, ec) - if err := db.SetExternalId(ec, userCred, iec.GetGlobalId()); err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.SetExternalId") - } - - { - // todo: 开启外网访问 - } - - if err := ec.SyncWithCloudElasticcache(ctx, userCred, provider, iec); err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.SyncWithCloudElasticcache") - } - - // sync accounts - { - iaccounts, err := iec.GetICloudElasticcacheAccounts() - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.GetICloudElasticcacheAccounts") - } - - result := models.ElasticcacheAccountManager.SyncElasticcacheAccounts(ctx, userCred, ec, iaccounts) - log.Debugf("aliyunRegionDriver.CreateElasticcache.SyncElasticcacheAccounts %s", result.Result()) - - account, err := ec.GetAdminAccount() - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.GetAdminAccount") - } - - err = account.SavePassword(params.Password) - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.SavePassword") - } - } - - // sync acl - { - iacls, err := iec.GetICloudElasticcacheAcls() - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.GetICloudElasticcacheAcls") - } - - result := models.ElasticcacheAclManager.SyncElasticcacheAcls(ctx, userCred, ec, iacls) - log.Debugf("aliyunRegionDriver.CreateElasticcache.SyncElasticcacheAcls %s", result.Result()) - } - - // sync parameters - { - iparams, err := iec.GetICloudElasticcacheParameters() - if err != nil { - return nil, errors.Wrap(err, "aliyunRegionDriver.CreateElasticcache.GetICloudElasticcacheParameters") - } - - result := models.ElasticcacheParameterManager.SyncElasticcacheParameters(ctx, userCred, ec, iparams) - log.Debugf("aliyunRegionDriver.CreateElasticcache.SyncElasticcacheParameters %s", result.Result()) - } - - return nil, nil - }) - return nil + return self.SManagedVirtualizationRegionDriver.ValidateCreateElasticcacheData(ctx, userCred, ownerId, input) } func (self *SAliyunRegionDriver) ValidateCreateElasticcacheAccountData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/regiondrivers/base.go b/pkg/compute/regiondrivers/base.go index 452ce905c4..2714134ecf 100644 --- a/pkg/compute/regiondrivers/base.go +++ b/pkg/compute/regiondrivers/base.go @@ -396,6 +396,10 @@ func (self *SBaseRegionDriver) RequestSyncDBInstanceBackupStatus(ctx context.Con return fmt.Errorf("Not Implement RequestSyncDBInstanceBackupStatus") } +func (self *SBaseRegionDriver) RequestCreateElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask, data *jsonutils.JSONDict) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestCreateElasticcache") +} + func (self *SBaseRegionDriver) RequestSyncElasticcacheStatus(ctx context.Context, userCred mcclient.TokenCredential, elasticcache *models.SElasticcache, task taskman.ITask) error { return fmt.Errorf("Not Implement RequestSyncElasticcacheStatus") } diff --git a/pkg/compute/regiondrivers/hcs.go b/pkg/compute/regiondrivers/hcs.go new file mode 100644 index 0000000000..6fddf7a1ea --- /dev/null +++ b/pkg/compute/regiondrivers/hcs.go @@ -0,0 +1,49 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regiondrivers + +import ( + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type SHCSRegionDriver struct { + SHuaWeiRegionDriver +} + +func init() { + driver := SHCSRegionDriver{} + models.RegisterRegionDriver(&driver) +} + +func (self *SHCSRegionDriver) GetProvider() string { + return api.CLOUD_PROVIDER_HCS +} + +func (self *SHCSRegionDriver) IsSupportedElasticcache() bool { + return true +} + +func (self *SHCSRegionDriver) IsSecurityGroupBelongVpc() bool { + return true +} + +func (self *SHCSRegionDriver) IsSupportedElasticcacheSecgroup() bool { + return true +} + +func (self *SHCSRegionDriver) GetMaxElasticcacheSecurityGroupCount() int { + return 1 +} diff --git a/pkg/compute/regiondrivers/huawei.go b/pkg/compute/regiondrivers/huawei.go index 3f03777459..5d2dee8a9a 100644 --- a/pkg/compute/regiondrivers/huawei.go +++ b/pkg/compute/regiondrivers/huawei.go @@ -20,7 +20,6 @@ import ( "fmt" "regexp" "strings" - "time" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -29,7 +28,6 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" - billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -40,9 +38,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" - "yunion.io/x/onecloud/pkg/util/choices" "yunion.io/x/onecloud/pkg/util/rand" - "yunion.io/x/onecloud/pkg/util/seclib2" ) type SHuaWeiRegionDriver struct { @@ -149,6 +145,7 @@ func (self *SHuaWeiRegionDriver) ValidateCreateLoadbalancerData(ctx context.Cont data.Set("network_type", jsonutils.NewString(api.LB_NETWORK_TYPE_VPC)) data.Set("cloudregion_id", jsonutils.NewString(region.GetId())) data.Set("vpc_id", jsonutils.NewString(vpc.GetId())) + data.Set("manager_id", jsonutils.NewString(vpc.ManagerId)) return self.SManagedVirtualizationRegionDriver.ValidateCreateLoadbalancerData(ctx, userCred, ownerId, data) } @@ -2345,225 +2342,19 @@ func validatorSlaveZones(ownerId mcclient.IIdentityProvider, regionId string, da return nil } -func ValidateElasticcacheSku(zoneId string, chargeType string, sku *models.SElasticcacheSku, network *models.SNetwork) error { - if sku.ZoneId != zoneId { - return httperrors.NewResourceNotFoundError("zone mismatch, elastic cache sku zone %s != %s", sku.ZoneId, zoneId) +func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { + if !utils.IsInStringArray(input.Engine, []string{"redis", "memcache"}) { + return nil, httperrors.NewInputParameterError("invalid engine %s", input.Engine) + } + if len(input.MaintainStartTime) > 0 && !utils.IsInStringArray(input.MaintainStartTime, []string{"22:00:00", "02:00:00", "06:00:00", "10:00:00", "14:00:00", "18:00:00"}) { + return nil, httperrors.NewInputParameterError("invalid maintain_start_time %s", input.MaintainStartTime) } - if network != nil { - if zone, _ := network.GetZone(); zone != nil && zone.Id != sku.ZoneId { - return httperrors.NewResourceNotFoundError("elastic cache sku zone (%s) and subnet zone (%s) mismatch", sku.ZoneId, zone.Id) - } + if input.CapacityMb == 0 { + return nil, httperrors.NewMissingParameterError("capacity_mb") } - if chargeType == billing_api.BILLING_TYPE_PREPAID { - if sku.PrepaidStatus != api.SkuStatusAvailable { - return httperrors.NewOutOfResourceError("sku %s is soldout", sku.Name) - } - } else { - if sku.PostpaidStatus != api.SkuStatusAvailable { - return httperrors.NewOutOfResourceError("sku %s is soldout", sku.Name) - } - } - - return nil -} - -func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) { - data := input.JSON(input) - zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId) - networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId) - // secgroupV := validators.NewModelIdOrNameValidator("security_group", "secgroup", ownerId) - - // todo: fix me - instanceTypeV := validators.NewModelIdOrNameValidator("instance_type", "elasticcachesku", ownerId) - chargeTypeV := validators.NewStringChoicesValidator("billing_type", choices.NewChoices(billing_api.BILLING_TYPE_PREPAID, billing_api.BILLING_TYPE_POSTPAID)) - networkTypeV := validators.NewStringChoicesValidator("network_type", choices.NewChoices(api.LB_NETWORK_TYPE_VPC, api.LB_NETWORK_TYPE_CLASSIC)).Default(api.LB_NETWORK_TYPE_VPC).Optional(true) - engineV := validators.NewStringChoicesValidator("engine", choices.NewChoices("redis", "memcache")) - engineVersionV := validators.NewStringChoicesValidator("engine_version", choices.NewChoices("3.0", "4.0", "5.0")) - privateIpV := validators.NewIPv4AddrValidator("private_ip").Optional(true) - maintainStartTimeV := validators.NewStringChoicesValidator("maintain_start_time", choices.NewChoices("22:00:00", "02:00:00", "06:00:00", "10:00:00", "14:00:00", "18:00:00")).Default("02:00:00").Optional(true) - - keyV := map[string]validators.IValidator{ - "zone": zoneV, - "billing_type": chargeTypeV, - "network_type": networkTypeV, - "network": networkV, - "instance_type": instanceTypeV, - // "security_group": secgroupV, - "engine": engineV, - "engine_version": engineVersionV, - "private_ip": privateIpV, - "mantain_start_time": maintainStartTimeV, - } - if err := RunValidators(keyV, data, false); err != nil { - return nil, err - } - - // validate password - if password, _ := data.GetString("password"); len(password) > 0 { - err := seclib2.ValidatePassword(password) - if err != nil { - return nil, err - } - } - - // validate sku - sku := instanceTypeV.Model.(*models.SElasticcacheSku) - zoneId, _ := data.GetString("zone_id") - billingType, _ := data.GetString("billing_type") - network := networkV.Model.(*models.SNetwork) - if err := ValidateElasticcacheSku(zoneId, billingType, sku, network); err != nil { - return nil, err - } else { - data.Set("instance_type", jsonutils.NewString(sku.InstanceSpec)) - data.Set("node_type", jsonutils.NewString(sku.NodeType)) - data.Set("local_category", jsonutils.NewString(sku.LocalCategory)) - data.Set("capacity_mb", jsonutils.NewInt(int64(sku.MemorySizeMB))) - } - - // validate slave zones - if len(input.SlaveZones) > 0 { - data.Set("slave_zones", jsonutils.NewString(strings.Join(input.SlaveZones, ","))) - } - if err := validatorSlaveZones(ownerId, zoneV.Model.(*models.SZone).GetCloudRegionId(), data, true); err != nil { - return nil, err - } - - // validate capacity - if capacityMB, err := data.Int("capacity_mb"); err != nil { - return nil, errors.Wrap(err, "invalid parameter capacity_mb") - } else { - // todo: fix me - // Redis引擎：单机和主备类型实例取值：2、4、8、16、32、64。集群实例规格支持64G、128G、256G、512G和1024G。 - data.Set("capacity_mb", jsonutils.NewInt(capacityMB)) - } - - // MaintainEndTime - // 开始时间必须为22:00:00、02:00:00、06:00:00、10:00:00、14:00:00和18:00:00。 - data.Remove("mantain_end_time") - if startTime, err := data.GetString("maintain_start_time"); err == nil { - maintainTimes := []string{"22:00:00", "02:00:00", "06:00:00", "10:00:00", "14:00:00", "18:00:00", "22:00:00"} - - for i := range maintainTimes { - if maintainTimes[i] == startTime { - data.Add(jsonutils.NewString(maintainTimes[i+1]), "mantain_end_time") - break - } - } - } - - // billing cycle - if billingType == billing_api.BILLING_TYPE_PREPAID { - billingCycle, err := data.GetString("billing_cycle") - if err != nil { - return nil, httperrors.NewMissingParameterError("billing_cycle") - } - - cycle, err := billing.ParseBillingCycle(billingCycle) - if err != nil { - return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", billingCycle) - } - - data.Set("billing_cycle", jsonutils.NewString(cycle.String())) - } - - vpc, _ := network.GetVpc() - if vpc == nil { - return nil, httperrors.NewNotFoundError("network %s related vpc not found", network.GetId()) - } - data.Set("vpc_id", jsonutils.NewString(vpc.Id)) - data.Set("manager_id", jsonutils.NewString(vpc.ManagerId)) - data.Set("cloudregion_id", jsonutils.NewString(sku.CloudregionId)) - return data, nil -} - -func (self *SHuaWeiRegionDriver) RequestCreateElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask, data *jsonutils.JSONDict) error { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - iRegion, err := ec.GetIRegion(ctx) - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.GetIRegion") - } - - provider := ec.GetCloudprovider() - if provider == nil { - return nil, errors.Wrap(httperrors.ErrInvalidStatus, "huaweiRegionDriver.CreateElasticcache.GetProvider") - } - - params, err := ec.GetCreateHuaweiElasticcacheParams(ctx, task.GetParams()) - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.GetCreateHuaweiElasticcacheParams") - } - - iec, err := iRegion.CreateIElasticcaches(params) - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.CreateIElasticcaches") - } - - err = cloudprovider.WaitStatusWithDelay(iec, api.ELASTIC_CACHE_STATUS_RUNNING, 30*time.Second, 15*time.Second, 600*time.Second) - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.WaitStatusWithDelay") - } - - ec.SetModelManager(models.ElasticcacheManager, ec) - if err := db.SetExternalId(ec, userCred, iec.GetGlobalId()); err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.SetExternalId") - } - - { - // todo: 开启外网访问 - } - - if err := ec.SyncWithCloudElasticcache(ctx, userCred, provider, iec); err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.SyncWithCloudElasticcache") - } - - // sync accounts - { - iaccounts, err := iec.GetICloudElasticcacheAccounts() - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.GetICloudElasticcacheAccounts") - } - - result := models.ElasticcacheAccountManager.SyncElasticcacheAccounts(ctx, userCred, ec, iaccounts) - log.Debugf("huaweiRegionDriver.CreateElasticcache.SyncElasticcacheAccounts %s", result.Result()) - - account, err := ec.GetAdminAccount() - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.GetAdminAccount") - } - - err = account.SavePassword(params.Password) - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.SavePassword") - } - } - - // sync acl - { - iacls, err := iec.GetICloudElasticcacheAcls() - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.GetICloudElasticcacheAcls") - } - - result := models.ElasticcacheAclManager.SyncElasticcacheAcls(ctx, userCred, ec, iacls) - log.Debugf("huaweiRegionDriver.CreateElasticcache.SyncElasticcacheAcls %s", result.Result()) - } - - // sync parameters - { - iparams, err := iec.GetICloudElasticcacheParameters() - if err != nil { - return nil, errors.Wrap(err, "huaweiRegionDriver.CreateElasticcache.GetICloudElasticcacheParameters") - } - - result := models.ElasticcacheParameterManager.SyncElasticcacheParameters(ctx, userCred, ec, iparams) - log.Debugf("huaweiRegionDriver.CreateElasticcache.SyncElasticcacheParameters %s", result.Result()) - } - - return nil, nil - }) - return nil + return self.SManagedVirtualizationRegionDriver.ValidateCreateElasticcacheData(ctx, userCred, ownerId, input) } func (self *SHuaWeiRegionDriver) ValidateCreateElasticcacheAccountData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/compute/regiondrivers/kvm.go b/pkg/compute/regiondrivers/kvm.go index f572e5703d..d7bc4d8341 100644 --- a/pkg/compute/regiondrivers/kvm.go +++ b/pkg/compute/regiondrivers/kvm.go @@ -1720,12 +1720,8 @@ func (self *SKVMRegionDriver) ValidateCacheSecgroup(ctx context.Context, userCre return fmt.Errorf("No need to cache secgroup for onecloud region") } -func (self *SKVMRegionDriver) RequestCreateElasticcache(ctx context.Context, userCred mcclient.TokenCredential, elasticcache *models.SElasticcache, task taskman.ITask, data *jsonutils.JSONDict) error { - return nil -} - -func (self *SKVMRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) { - return input.JSON(input), nil +func (self *SKVMRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { + return input, httperrors.NewNotSupportedError("Not support create elasticcache") } func (self *SKVMRegionDriver) RequestRestartElasticcache(ctx context.Context, userCred mcclient.TokenCredential, elasticcache *models.SElasticcache, task taskman.ITask) error { diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index c67b319afd..7225faf8e5 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -1874,20 +1874,158 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateDBInstanceFromBacku return nil } -func (self *SManagedVirtualizationRegionDriver) RequestCreateElasticcache(ctx context.Context, userCred mcclient.TokenCredential, elasticcache *models.SElasticcache, task taskman.ITask, data *jsonutils.JSONDict) error { - task.ScheduleRun(nil) +func (self *SManagedVirtualizationRegionDriver) RequestCreateElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask, data *jsonutils.JSONDict) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iRegion, err := ec.GetIRegion(ctx) + if err != nil { + return nil, errors.Wrap(err, "GetIRegion") + } + + iprovider, err := db.FetchById(models.CloudproviderManager, ec.GetCloudproviderId()) + if err != nil { + return nil, errors.Wrap(err, "GetProvider") + } + vpc, err := ec.GetVpc() + if err != nil { + return nil, errors.Wrapf(err, "GetVpc") + } + + networkObj, err := db.FetchById(models.NetworkManager, ec.NetworkId) + if err != nil { + return nil, errors.Wrap(err, "GetNetwork") + } + network := networkObj.(*models.SNetwork) + + params := &cloudprovider.SCloudElasticCacheInput{ + InstanceType: ec.InstanceType, + InstanceName: ec.Name, + Engine: ec.Engine, + EngineVersion: ec.EngineVersion, + PrivateIpAddress: ec.PrivateIpAddr, + CapacityGB: int64(ec.CapacityMB / 1024), + NodeType: ec.NodeType, + NetworkType: ec.NetworkType, + VpcId: vpc.ExternalId, + NetworkId: network.ExternalId, + MaintainBegin: ec.MaintainStartTime, + MaintainEnd: ec.MaintainEndTime, + } + if ec.BillingType == billing_api.BILLING_TYPE_PREPAID { + bc, err := billing.ParseBillingCycle(ec.BillingCycle) + if err != nil { + return nil, errors.Wrapf(err, "ParseBillingCycle(%s)", ec.BillingCycle) + } + bc.AutoRenew = ec.AutoRenew + params.BillingCycle = &bc + } + params.Tags, _ = ec.GetAllUserMetadata() + + zone, err := ec.GetZone() + if zone != nil { + izone, err := iRegion.GetIZoneById(zone.ExternalId) + if err != nil { + return nil, errors.Wrapf(err, "GetIZoneById(%s)", zone.ExternalId) + } + params.ZoneIds = []string{izone.GetId()} + } + slaveZones, err := ec.GetSlaveZones() + if err != nil { + return nil, errors.Wrapf(err, "GetSlaveZones") + } + for i := range slaveZones { + izone, err := iRegion.GetIZoneById(slaveZones[i].ExternalId) + if err != nil { + return nil, errors.Wrapf(err, "GetSlaveZoneBy(%s)", slaveZones[i].ExternalId) + } + if !utils.IsInStringArray(izone.GetId(), params.ZoneIds) { + params.ZoneIds = append(params.ZoneIds, izone.GetId()) + } + } + + params.Password, _ = data.GetString("password") + data.Unmarshal(¶ms.SecurityGroupIds, "ext_secgroup_ids") + + provider := iprovider.(*models.SCloudprovider) + params.ProjectId, err = provider.SyncProject(ctx, userCred, ec.ProjectId) + if err != nil { + log.Errorf("failed to sync project %s for create %s elastic cache %s error: %v", ec.ProjectId, provider.Provider, ec.Name, err) + } + + iec, err := iRegion.CreateIElasticcaches(params) + if err != nil { + return nil, errors.Wrap(err, "CreateIElasticcaches") + } + + err = db.SetExternalId(ec, userCred, iec.GetGlobalId()) + if err != nil { + return nil, errors.Wrap(err, "SetExternalId") + } + + err = cloudprovider.WaitStatusWithDelay(iec, api.ELASTIC_CACHE_STATUS_RUNNING, 30*time.Second, 15*time.Second, 10*time.Minute) + if err != nil { + return nil, errors.Wrap(err, "WaitStatusWithDelay") + } + + err = ec.SyncWithCloudElasticcache(ctx, userCred, provider, iec) + if err != nil { + return nil, errors.Wrap(err, "SyncWithCloudElasticcache") + } + + // sync accounts + { + iaccounts, err := iec.GetICloudElasticcacheAccounts() + if err != nil { + return nil, errors.Wrap(err, "GetICloudElasticcacheAccounts") + } + + result := models.ElasticcacheAccountManager.SyncElasticcacheAccounts(ctx, userCred, ec, iaccounts) + log.Infof("SyncElasticcacheAccounts %s", result.Result()) + + account, err := ec.GetAdminAccount() + if err != nil { + return nil, errors.Wrap(err, "GetAdminAccount") + } + + err = account.SavePassword(params.Password) + if err != nil { + return nil, errors.Wrap(err, "SavePassword") + } + } + + // sync acl + { + iacls, err := iec.GetICloudElasticcacheAcls() + if err != nil { + return nil, errors.Wrap(err, "GetICloudElasticcacheAcls") + } + + result := models.ElasticcacheAclManager.SyncElasticcacheAcls(ctx, userCred, ec, iacls) + log.Infof("SyncElasticcacheAcls %s", result.Result()) + } + + // sync parameters + { + iparams, err := iec.GetICloudElasticcacheParameters() + if err != nil { + return nil, errors.Wrap(err, "GetICloudElasticcacheParameters") + } + + result := models.ElasticcacheParameterManager.SyncElasticcacheParameters(ctx, userCred, ec, iparams) + log.Infof("SyncElasticcacheParameters %s", result.Result()) + } + return nil, nil + }) return nil } -func (self *SManagedVirtualizationRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) { - m := jsonutils.NewDict() - m.Set("manager_id", jsonutils.NewString(input.ManagerId)) - _, err := self.ValidateManagerId(ctx, userCred, m) - if err != nil { - return nil, errors.Wrap(err, "ValidateManagerId") +func (self *SManagedVirtualizationRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { + if len(input.ManagerId) == 0 { + return nil, httperrors.NewMissingParameterError("manager_id") } - - return input.JSON(input), nil + if len(input.NetworkType) == 0 { + input.NetworkType = api.LB_NETWORK_TYPE_VPC + } + return input, nil } func (self *SManagedVirtualizationRegionDriver) RequestRestartElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask) error { @@ -2994,7 +3132,47 @@ func (self *SManagedVirtualizationRegionDriver) RequestRemoteUpdateElasticcache( } func (self *SManagedVirtualizationRegionDriver) RequestSyncSecgroupsForElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask) error { - return fmt.Errorf("Not Implement RequestSyncSecgroupsForElasticcache") + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + // sync secgroups to cloud + secgroupExternalIds := []string{} + { + ess, err := ec.GetElasticcacheSecgroups() + if err != nil { + return nil, errors.Wrap(err, "GetElasticcacheSecgroups") + } + + provider := ec.GetCloudprovider() + if provider == nil { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "GetCloudprovider") + } + + vpc, err := ec.GetVpc() + if err != nil { + return nil, errors.Wrapf(err, "GetVpc") + } + region, err := vpc.GetRegion() + if err != nil { + return nil, errors.Wrapf(err, "GetRegion") + } + vpcId, err := self.GetSecurityGroupVpcId(ctx, userCred, region, nil, vpc, false) + if err != nil { + return nil, errors.Wrap(err, "GetSecurityGroupVpcId") + } + + for i := range ess { + externalId, err := self.RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, ess[i].GetSecGroup(), "", "redis", true) + if err != nil { + return nil, errors.Wrap(err, "RequestSyncSecurityGroup") + } + secgroupExternalIds = append(secgroupExternalIds, externalId) + } + } + + ret := jsonutils.NewDict() + ret.Set("ext_secgroup_ids", jsonutils.NewStringArray(secgroupExternalIds)) + return ret, nil + }) + return nil } func (self *SManagedVirtualizationRegionDriver) RequestRenewElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, bc billing.SBillingCycle) (time.Time, error) { diff --git a/pkg/compute/regiondrivers/qcloud.go b/pkg/compute/regiondrivers/qcloud.go index ac23ffeff1..31208c22ef 100644 --- a/pkg/compute/regiondrivers/qcloud.go +++ b/pkg/compute/regiondrivers/qcloud.go @@ -19,7 +19,6 @@ import ( "fmt" "regexp" "strconv" - "strings" "time" "yunion.io/x/jsonutils" @@ -29,7 +28,6 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" - billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -1481,203 +1479,18 @@ func (self *SQcloudRegionDriver) GetMaxElasticcacheSecurityGroupCount() int { return 10 } -func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input api.ElasticcacheCreateInput) (*jsonutils.JSONDict, error) { - data := input.JSON(input) - zoneV := validators.NewModelIdOrNameValidator("zone", "zone", ownerId) - networkV := validators.NewModelIdOrNameValidator("network", "network", ownerId) - instanceTypeV := validators.NewModelIdOrNameValidator("instance_type", "elasticcachesku", ownerId) - chargeTypeV := validators.NewStringChoicesValidator("billing_type", choices.NewChoices(billing_api.BILLING_TYPE_PREPAID, billing_api.BILLING_TYPE_POSTPAID)) - networkTypeV := validators.NewStringChoicesValidator("network_type", choices.NewChoices(api.LB_NETWORK_TYPE_VPC)).Default(api.LB_NETWORK_TYPE_VPC).Optional(true) - engineVersionV := validators.NewStringChoicesValidator("engine_version", choices.NewChoices("2.8", "3.0", "3.2", "4.0", "5.0")) - - keyV := map[string]validators.IValidator{ - "zone": zoneV, - "billing_type": chargeTypeV, - "network_type": networkTypeV, - "network": networkV, - "instance_type": instanceTypeV, - "engine_version": engineVersionV, +func (self *SQcloudRegionDriver) ValidateCreateElasticcacheData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.ElasticcacheCreateInput) (*api.ElasticcacheCreateInput, error) { + if len(input.NetworkType) == 0 { + input.NetworkType = api.LB_NETWORK_TYPE_VPC } - if err := RunValidators(keyV, data, false); err != nil { - return nil, err - } - - // validate password - if password, _ := data.GetString("password"); len(password) > 0 { - err := seclib2.ValidatePassword(password) - if err != nil { - return nil, err - } - } - - zoneId, _ := data.GetString("zone_id") - billingType, _ := data.GetString("billing_type") - // validate sku - sku := instanceTypeV.Model.(*models.SElasticcacheSku) - network := networkV.Model.(*models.SNetwork) - if err := ValidateElasticcacheSku(zoneId, billingType, sku, network); err != nil { - return nil, err - } else { - data.Set("instance_type", jsonutils.NewString(sku.InstanceSpec)) - data.Set("node_type", jsonutils.NewString(sku.NodeType)) - data.Set("local_category", jsonutils.NewString(sku.LocalCategory)) - data.Set("capacity_mb", jsonutils.NewInt(int64(sku.MemorySizeMB))) - } - - // validate slave zones - if len(input.SlaveZones) > 0 { - if len(input.SlaveZones) < sku.ReplicasNum { - padding := make([]string, sku.ReplicasNum-len(input.SlaveZones)) - for i := range padding { - padding[i] = zoneId - } - - input.SlaveZones = append(input.SlaveZones, padding...) - } - - data.Set("slave_zones", jsonutils.NewString(strings.Join(input.SlaveZones, ","))) - } - if err := validatorSlaveZones(ownerId, zoneV.Model.(*models.SZone).GetCloudRegionId(), data, true); err != nil { - return nil, err - } - - sz, _ := data.GetString("slave_zones") - if len(strings.Split(sz, ",")) > sku.ReplicasNum { - return nil, fmt.Errorf("the number of slave zones can not beyond redis replicas number") - } - - // validate secgroups - secgroups := []string{} - err := data.Unmarshal(&secgroups, "secgroup_ids") - if err != nil { - log.Debugf("Unmarshal.security_groups %s", err) - data.Set("secgroup_ids", jsonutils.NewArray(jsonutils.NewString(api.SECGROUP_DEFAULT_ID))) - secgroups = []string{api.SECGROUP_DEFAULT_ID} - } - - if len(secgroups) == 0 || len(secgroups) > 10 { - return nil, errors.Wrap(err, "secgroups id quantity should between 1 and 10.") - } - - _, err = models.CheckingSecgroupIds(ctx, userCred, secgroups) - if err != nil { - return nil, errors.Wrap(err, "CheckingSecgroupIds") - } - - // billing cycle - if billingType == billing_api.BILLING_TYPE_PREPAID { - billingCycle, err := data.GetString("billing_cycle") - if err != nil { - return nil, httperrors.NewMissingParameterError("billing_cycle") - } - - cycle, err := billing.ParseBillingCycle(billingCycle) - if err != nil { - return nil, httperrors.NewInputParameterError("invalid billing_cycle %s", billingCycle) - } - - data.Set("billing_cycle", jsonutils.NewString(cycle.String())) - } - - vpc, _ := network.GetVpc() - if vpc == nil { - return nil, httperrors.NewNotFoundError("network %s related vpc not found", network.GetId()) - } - data.Set("engine", jsonutils.NewString("redis")) - data.Set("vpc_id", jsonutils.NewString(vpc.Id)) - data.Set("manager_id", jsonutils.NewString(vpc.ManagerId)) - data.Set("cloudregion_id", jsonutils.NewString(vpc.CloudregionId)) - return data, nil + input.Engine = "redis" + return self.SManagedVirtualizationRegionDriver.ValidateCreateElasticcacheData(ctx, userCred, ownerId, input) } func (self *SQcloudRegionDriver) IsSupportedElasticcache() bool { return true } -func (self *SQcloudRegionDriver) RequestCreateElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask, data *jsonutils.JSONDict) error { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - iRegion, err := ec.GetIRegion(ctx) - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.GetIRegion") - } - - provider := ec.GetCloudprovider() - if provider == nil { - return nil, errors.Wrap(httperrors.ErrInvalidStatus, "qcloudRegionDriver.CreateElasticcache.GetProvider") - } - - secgroups := []string{} - err = data.Unmarshal(&secgroups, "ext_secgroup_ids") - if err != nil { - return nil, errors.Wrap(err, "Unmarshal.ext_secgroup_ids") - } - - params, err := ec.GetCreateQCloudElasticcacheParams(ctx, task.GetParams()) - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.GetCreateHuaweiElasticcacheParams") - } - params.SecurityGroupIds = secgroups - params.ProjectId, err = provider.SyncProject(ctx, userCred, ec.ProjectId) - if err != nil { - log.Errorf("failed to sync project %s for create %s elastic cache %s error: %v", ec.ProjectId, provider.Provider, ec.Name, err) - } - - iec, err := iRegion.CreateIElasticcaches(params) - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.CreateIElasticcaches") - } - - err = cloudprovider.WaitStatusWithDelay(iec, api.ELASTIC_CACHE_STATUS_RUNNING, 30*time.Second, 15*time.Second, 900*time.Second) - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.WaitStatusWithDelay") - } - - ec.SetModelManager(models.ElasticcacheManager, ec) - if err := db.SetExternalId(ec, userCred, iec.GetGlobalId()); err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.SetExternalId") - } - - if err := ec.SyncWithCloudElasticcache(ctx, userCred, provider, iec); err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.SyncWithCloudElasticcache") - } - - // sync accounts - { - iaccounts, err := iec.GetICloudElasticcacheAccounts() - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.GetICloudElasticcacheAccounts") - } - - result := models.ElasticcacheAccountManager.SyncElasticcacheAccounts(ctx, userCred, ec, iaccounts) - log.Debugf("qcloudRegionDriver.CreateElasticcache.SyncElasticcacheAccounts %s", result.Result()) - - account, err := ec.GetAdminAccount() - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.GetAdminAccount") - } - - err = account.SavePassword(params.Password) - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.SavePassword") - } - } - - // sync parameters - { - iparams, err := iec.GetICloudElasticcacheParameters() - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.CreateElasticcache.GetICloudElasticcacheParameters") - } - - result := models.ElasticcacheParameterManager.SyncElasticcacheParameters(ctx, userCred, ec, iparams) - log.Debugf("qcloudRegionDriver.CreateElasticcache.SyncElasticcacheParameters %s", result.Result()) - } - - return nil, nil - }) - return nil -} - func (self *SQcloudRegionDriver) GetSecurityGroupPublicScope(service string) rbacutils.TRbacScope { if service == "redis" { return rbacutils.ScopeProject @@ -1685,49 +1498,6 @@ func (self *SQcloudRegionDriver) GetSecurityGroupPublicScope(service string) rba return rbacutils.ScopeSystem } -func (self *SQcloudRegionDriver) RequestSyncSecgroupsForElasticcache(ctx context.Context, userCred mcclient.TokenCredential, ec *models.SElasticcache, task taskman.ITask) error { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - // sync secgroups to cloud - secgroupExternalIds := []string{} - { - ess, err := ec.GetElasticcacheSecgroups() - if err != nil { - return nil, errors.Wrap(err, "qcloudRegionDriver.GetElasticcacheSecgroups") - } - - provider := ec.GetCloudprovider() - if provider == nil { - return nil, errors.Wrap(httperrors.ErrInvalidStatus, "qcloudRegionDriver.GetCloudprovider") - } - - extProjectId, err := provider.SyncProject(ctx, userCred, ec.ProjectId) - if err != nil { - return nil, fmt.Errorf("failed to sync project %s for create %s elastic cache %s error: %v", ec.ProjectId, provider.Provider, ec.Name, err) - } - - vpc, _ := ec.GetVpc() - region, _ := vpc.GetRegion() - vpcId, err := self.GetSecurityGroupVpcId(ctx, userCred, region, nil, vpc, false) - if err != nil { - return nil, errors.Wrap(err, "GetSecurityGroupVpcId") - } - - for i := range ess { - externalId, err := self.RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, ess[i].GetSecGroup(), extProjectId, "redis", true) - if err != nil { - return nil, errors.Wrap(err, "RequestSyncSecurityGroup") - } - secgroupExternalIds = append(secgroupExternalIds, externalId) - } - } - - ret := jsonutils.NewDict() - ret.Set("ext_secgroup_ids", jsonutils.NewStringArray(secgroupExternalIds)) - return ret, nil - }) - return nil -} - func (self *SQcloudRegionDriver) ValidateCreateElasticcacheAccountData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { elasticCacheV := validators.NewModelIdOrNameValidator("elasticcache", "elasticcache", ownerId) accountTypeV := validators.NewStringChoicesValidator("account_type", choices.NewChoices("normal")).Default("normal") diff --git a/pkg/compute/tasks/elasticcache_create_task.go b/pkg/compute/tasks/elasticcache_create_task.go index 5f311f88c0..dfa250ab8e 100644 --- a/pkg/compute/tasks/elasticcache_create_task.go +++ b/pkg/compute/tasks/elasticcache_create_task.go @@ -19,6 +19,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -36,23 +37,23 @@ func init() { taskman.RegisterTask(ElasticcacheCreateTask{}) } -func (self *ElasticcacheCreateTask) taskFail(ctx context.Context, elasticcache *models.SElasticcache, reason jsonutils.JSONObject) { - elasticcache.SetStatus(self.GetUserCred(), api.ELASTIC_CACHE_STATUS_CREATE_FAILED, reason.String()) - db.OpsLog.LogEvent(elasticcache, db.ACT_ALLOCATE_FAIL, reason, self.UserCred) - logclient.AddActionLogWithStartable(self, elasticcache, logclient.ACT_CREATE, reason, self.UserCred, false) +func (self *ElasticcacheCreateTask) taskFail(ctx context.Context, elasticcache *models.SElasticcache, err error) { + elasticcache.SetStatus(self.GetUserCred(), api.ELASTIC_CACHE_STATUS_CREATE_FAILED, err.Error()) + db.OpsLog.LogEvent(elasticcache, db.ACT_ALLOCATE_FAIL, err, self.UserCred) + logclient.AddActionLogWithStartable(self, elasticcache, logclient.ACT_CREATE, err, self.UserCred, false) notifyclient.EventNotify(ctx, self.GetUserCred(), notifyclient.SEventNotifyParam{ Obj: elasticcache, Action: notifyclient.ActionCreate, IsFail: true, }) - self.SetStageFailed(ctx, reason) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) } func (self *ElasticcacheCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { elasticcache := obj.(*models.SElasticcache) - region, _ := elasticcache.GetRegion() - if region == nil { - self.taskFail(ctx, elasticcache, jsonutils.NewString(fmt.Sprintf("failed to find region for elastic cache %s", elasticcache.GetName()))) + region, err := elasticcache.GetRegion() + if err != nil { + self.taskFail(ctx, elasticcache, errors.Wrapf(err, "GetRegion")) return } @@ -60,22 +61,16 @@ func (self *ElasticcacheCreateTask) OnInit(ctx context.Context, obj db.IStandalo self.SetStage("OnSyncSecurityGroupComplete", data.(*jsonutils.JSONDict)) if region.GetDriver().IsSupportedElasticcacheSecgroup() { secgroups := []string{} - err := self.GetParams().Unmarshal(&secgroups, "secgroup_ids") - if err != nil { - notes := fmt.Sprintf("Unmarshal.secgroup_ids %s", err) - self.taskFail(ctx, elasticcache, jsonutils.NewString(notes)) - return - } + self.GetParams().Unmarshal(&secgroups, "secgroup_ids") secgroupInput := api.ElasticcacheSecgroupsInput{SecgroupIds: secgroups} _, err = elasticcache.ProcessElasticcacheSecgroupsInput(ctx, self.UserCred, "set", &secgroupInput) if err != nil { - notes := fmt.Sprintf("ProcessElasticcacheSecgroupsInput %s", err) - self.taskFail(ctx, elasticcache, jsonutils.NewString(notes)) + self.taskFail(ctx, elasticcache, errors.Wrapf(err, "ProcessElasticcacheSecgroupsInput")) return } if err := region.GetDriver().RequestSyncSecgroupsForElasticcache(ctx, self.UserCred, elasticcache, self); err != nil { - self.taskFail(ctx, elasticcache, jsonutils.NewString(err.Error())) + self.taskFail(ctx, elasticcache, errors.Wrapf(err, "RequestSyncSecgroupsForElasticcache")) return } } else { @@ -84,22 +79,26 @@ func (self *ElasticcacheCreateTask) OnInit(ctx context.Context, obj db.IStandalo } func (self *ElasticcacheCreateTask) OnSyncSecurityGroupComplete(ctx context.Context, elasticcache *models.SElasticcache, data jsonutils.JSONObject) { - region, _ := elasticcache.GetRegion() + region, err := elasticcache.GetRegion() + if err != nil { + self.taskFail(ctx, elasticcache, errors.Wrapf(err, "GetRegion")) + return + } self.SetStage("OnElasticcacheCreateComplete", nil) - if err := region.GetDriver().RequestCreateElasticcache(ctx, self.GetUserCred(), elasticcache, self, data.(*jsonutils.JSONDict)); err != nil { - self.taskFail(ctx, elasticcache, jsonutils.NewString(err.Error())) + err = region.GetDriver().RequestCreateElasticcache(ctx, self.GetUserCred(), elasticcache, self, data.(*jsonutils.JSONDict)) + if err != nil { + self.taskFail(ctx, elasticcache, errors.Wrapf(err, "RequestCreateElasticcache")) return } } func (self *ElasticcacheCreateTask) OnSyncSecurityGroupCompleteFailed(ctx context.Context, elasticcache *models.SElasticcache, reason jsonutils.JSONObject) { - self.taskFail(ctx, elasticcache, reason) + self.taskFail(ctx, elasticcache, fmt.Errorf(reason.String())) } func (self *ElasticcacheCreateTask) OnElasticcacheCreateComplete(ctx context.Context, elasticcache *models.SElasticcache, data jsonutils.JSONObject) { elasticcache.SetStatus(self.GetUserCred(), api.ELASTIC_CACHE_STATUS_RUNNING, "") logclient.AddActionLogWithStartable(self, elasticcache, logclient.ACT_CREATE, "", self.UserCred, true) - // notifyclient.NotifyWebhook(ctx, self.UserCred, elasticcache, notifyclient.ActionCreate) notifyclient.EventNotify(ctx, self.UserCred, notifyclient.SEventNotifyParam{ Obj: elasticcache, Action: notifyclient.ActionCreate, @@ -108,5 +107,5 @@ func (self *ElasticcacheCreateTask) OnElasticcacheCreateComplete(ctx context.Con } func (self *ElasticcacheCreateTask) OnElasticcacheCreateCompleteFailed(ctx context.Context, elasticcache *models.SElasticcache, reason jsonutils.JSONObject) { - self.taskFail(ctx, elasticcache, reason) + self.taskFail(ctx, elasticcache, fmt.Errorf(reason.String())) } diff --git a/pkg/compute/tasks/elasticcache_syncsecgroups_task.go b/pkg/compute/tasks/elasticcache_syncsecgroups_task.go index aea867d7b4..b0aceca0ee 100644 --- a/pkg/compute/tasks/elasticcache_syncsecgroups_task.go +++ b/pkg/compute/tasks/elasticcache_syncsecgroups_task.go @@ -19,6 +19,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -35,9 +36,9 @@ func init() { taskman.RegisterTask(ElasticcacheSyncsecgroupsTask{}) } -func (self *ElasticcacheSyncsecgroupsTask) taskFailed(ctx context.Context, cache *models.SElasticcache, err jsonutils.JSONObject) { - cache.SetStatus(self.GetUserCred(), api.ELASTIC_CACHE_STATUS_SYNC_FAILED, err.String()) - self.SetStageFailed(ctx, err) +func (self *ElasticcacheSyncsecgroupsTask) taskFailed(ctx context.Context, cache *models.SElasticcache, err error) { + cache.SetStatus(self.GetUserCred(), api.ELASTIC_CACHE_STATUS_SYNC_FAILED, err.Error()) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) db.OpsLog.LogEvent(cache, db.ACT_SYNC_CONF, cache.GetShortDesc(ctx), self.GetUserCred()) logclient.AddActionLogWithContext(ctx, cache, logclient.ACT_SYNC_CONF, err, self.UserCred, false) } @@ -45,16 +46,16 @@ func (self *ElasticcacheSyncsecgroupsTask) taskFailed(ctx context.Context, cache func (self *ElasticcacheSyncsecgroupsTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { cache := obj.(*models.SElasticcache) - region, _ := cache.GetRegion() - if region == nil { - self.taskFailed(ctx, cache, jsonutils.NewString(fmt.Sprintf("failed to found cloudregion for elasticcache %s(%s)", cache.Name, cache.Id))) + region, err := cache.GetRegion() + if err != nil { + self.taskFailed(ctx, cache, errors.Wrapf(err, "GetRegion")) return } self.SetStage("OnElasticcacheSyncSecgroupsComplete", nil) - err := region.GetDriver().RequestSyncSecgroupsForElasticcache(ctx, self.GetUserCred(), cache, self) + err = region.GetDriver().RequestSyncSecgroupsForElasticcache(ctx, self.GetUserCred(), cache, self) if err != nil { - self.taskFailed(ctx, cache, jsonutils.NewString(err.Error())) + self.taskFailed(ctx, cache, errors.Wrapf(err, "RequestSyncSecgroupsForElasticcache")) return } } @@ -65,25 +66,25 @@ func (self *ElasticcacheSyncsecgroupsTask) OnElasticcacheSyncSecgroupsComplete(c secgroups := []string{} err := data.Unmarshal(&secgroups, "ext_secgroup_ids") if err != nil { - self.taskFailed(ctx, cache, jsonutils.NewString(err.Error())) + self.taskFailed(ctx, cache, errors.Wrapf(err, "get ext_secgroup_ids")) return } iregion, err := cache.GetIRegion(ctx) if err != nil { - self.taskFailed(ctx, cache, jsonutils.NewString(err.Error())) + self.taskFailed(ctx, cache, errors.Wrapf(err, "GetIRegion")) return } iec, err := iregion.GetIElasticcacheById(cache.GetExternalId()) if err != nil { - self.taskFailed(ctx, cache, jsonutils.NewString(err.Error())) + self.taskFailed(ctx, cache, errors.Wrapf(err, "GetIElasticcacheById")) return } err = iec.UpdateSecurityGroups(secgroups) if err != nil { - self.taskFailed(ctx, cache, jsonutils.NewString(err.Error())) + self.taskFailed(ctx, cache, errors.Wrapf(err, "UpdateSecurityGroups")) return } @@ -92,5 +93,5 @@ func (self *ElasticcacheSyncsecgroupsTask) OnElasticcacheSyncSecgroupsComplete(c } func (self *ElasticcacheSyncsecgroupsTask) OnElasticcacheSyncSecgroupsCompleteFailed(ctx context.Context, cache *models.SElasticcache, data jsonutils.JSONObject) { - self.taskFailed(ctx, cache, data) + self.taskFailed(ctx, cache, fmt.Errorf(data.String())) } diff --git a/pkg/compute/tasks/loadbalancer_listener_create_task.go b/pkg/compute/tasks/loadbalancer_listener_create_task.go index b6e86a223a..48aa1de211 100644 --- a/pkg/compute/tasks/loadbalancer_listener_create_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_create_task.go @@ -38,7 +38,7 @@ func init() { func getOnLoadbalancerListenerCreateCompleteFunc(provider string) func(ctx context.Context, lblis *models.SLoadbalancerListener, data jsonutils.JSONObject, self *LoadbalancerListenerCreateTask) { switch provider { - case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO: + case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS: return onHuaweiLoadbalancerListenerCreateComplete case api.CLOUD_PROVIDER_OPENSTACK: return onOpenstackLoadbalancerListenerCreateComplete diff --git a/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go b/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go index b8f66e48c7..7d46e21319 100644 --- a/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go +++ b/pkg/compute/tasks/loadbalancer_listener_rule_create_task.go @@ -37,7 +37,7 @@ func init() { func getOnPrepareLoadbalancerBackendgroupFunc(provider string) func(ctx context.Context, region *models.SCloudregion, lbr *models.SLoadbalancerListenerRule, data jsonutils.JSONObject, self *LoadbalancerListenerRuleCreateTask) { switch provider { - case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO: + case api.CLOUD_PROVIDER_HUAWEI, api.CLOUD_PROVIDER_HCSO, api.CLOUD_PROVIDER_HCS: return onHuaiweiPrepareLoadbalancerBackendgroup case api.CLOUD_PROVIDER_AWS: return onAwsPrepareLoadbalancerBackendgroup diff --git a/pkg/mcclient/options/cloudaccounts.go b/pkg/mcclient/options/cloudaccounts.go index e354e33277..dc8c6e9d1a 100644 --- a/pkg/mcclient/options/cloudaccounts.go +++ b/pkg/mcclient/options/cloudaccounts.go @@ -306,6 +306,18 @@ func (opts *SHCSOAccountCreateOptions) Params() (jsonutils.JSONObject, error) { return params, nil } +type SHCSAccountCreateOptions struct { + SCloudAccountCreateBaseOptions + AuthURL string `help:"Hcs auth_url" positional:"true" json:"auth_url"` + SAccessKeyCredential +} + +func (opts *SHCSAccountCreateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(opts) + params.(*jsonutils.JSONDict).Add(jsonutils.NewString("HCS"), "provider") + return params, nil +} + type SUcloudCloudAccountCreateOptions struct { SCloudAccountCreateBaseOptions SAccessKeyCredential @@ -505,6 +517,15 @@ func (opts *SHCSOAccountUpdateCredentialOptions) Params() (jsonutils.JSONObject, return jsonutils.Marshal(opts), nil } +type SHCSAccountUpdateCredentialOptions struct { + SCloudAccountIdOptions + SAccessKeyCredential +} + +func (opts *SHCSAccountUpdateCredentialOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + type SUcloudCloudAccountUpdateCredentialOptions struct { SCloudAccountIdOptions SAccessKeyCredential @@ -854,6 +875,14 @@ func (opts *SHCSOAccountUpdateOptions) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(opts), nil } +type SHCSAccountUpdateOptions struct { + SCloudAccountUpdateBaseOptions +} + +func (opts *SHCSAccountUpdateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + type SUcloudCloudAccountUpdateOptions struct { SCloudAccountUpdateBaseOptions } diff --git a/pkg/multicloud/aliyun/elasticcache_instance.go b/pkg/multicloud/aliyun/elasticcache_instance.go index fdd548223e..ccaf2bfe75 100644 --- a/pkg/multicloud/aliyun/elasticcache_instance.go +++ b/pkg/multicloud/aliyun/elasticcache_instance.go @@ -26,7 +26,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - billingapi "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" @@ -642,7 +641,12 @@ func (self *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheIn } if len(ec.NodeType) > 0 { - params["NodeType"] = ec.NodeType + switch ec.NodeType { + case "single": + params["NodeType"] = "STAND_ALONE" + case "double": + params["NodeType"] = "MASTER_SLAVE" + } } if len(ec.ProjectId) > 0 { @@ -652,25 +656,24 @@ func (self *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheIn params["NetworkType"] = ec.NetworkType params["VpcId"] = ec.VpcId params["VSwitchId"] = ec.NetworkId - params["ChargeType"] = ec.ChargeType - if strings.ToLower(ec.ChargeType) == billingapi.BILLING_TYPE_PREPAID && ec.BC != nil { - if ec.BC.GetMonths() >= 1 && ec.BC.GetMonths() <= 9 { - params["Period"] = strconv.Itoa(ec.BC.GetMonths()) - } else if ec.BC.GetMonths() == 12 || ec.BC.GetMonths() == 24 || ec.BC.GetMonths() == 36 { - params["Period"] = strconv.Itoa(ec.BC.GetMonths()) + params["ChargeType"] = "PostPaid" + if ec.BillingCycle != nil { + params["ChargeType"] = "PrePaid" + if ec.BillingCycle.GetMonths() >= 1 && ec.BillingCycle.GetMonths() <= 9 { + params["Period"] = strconv.Itoa(ec.BillingCycle.GetMonths()) + } else if ec.BillingCycle.GetMonths() == 12 || ec.BillingCycle.GetMonths() == 24 || ec.BillingCycle.GetMonths() == 36 { + params["Period"] = strconv.Itoa(ec.BillingCycle.GetMonths()) } else { return nil, fmt.Errorf("region.CreateIElasticcaches invalid billing cycle.reqired month (1~9) or year(1~3)") } } - ret := &SElasticcache{} + ret := &SElasticcache{region: self} err := DoAction(self.kvsRequest, "CreateInstance", params, []string{}, ret) if err != nil { return nil, errors.Wrap(err, "region.CreateIElasticcaches") } self.SetResourceTags(ALIYUN_SERVICE_KVS, "INSTANCE", ret.InstanceID, ec.Tags, true) - - ret.region = self return ret, nil } diff --git a/pkg/multicloud/apsara/elasticcache_instance.go b/pkg/multicloud/apsara/elasticcache_instance.go index 75873bc250..64701371be 100644 --- a/pkg/multicloud/apsara/elasticcache_instance.go +++ b/pkg/multicloud/apsara/elasticcache_instance.go @@ -639,11 +639,11 @@ func (self *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheIn params["VpcId"] = ec.VpcId params["VSwitchId"] = ec.NetworkId params["ChargeType"] = ec.ChargeType - if strings.ToLower(ec.ChargeType) == billingapi.BILLING_TYPE_PREPAID && ec.BC != nil { - if ec.BC.GetMonths() >= 1 && ec.BC.GetMonths() <= 9 { - params["Period"] = strconv.Itoa(ec.BC.GetMonths()) - } else if ec.BC.GetMonths() == 12 || ec.BC.GetMonths() == 24 || ec.BC.GetMonths() == 36 { - params["Period"] = strconv.Itoa(ec.BC.GetMonths()) + if strings.ToLower(ec.ChargeType) == billingapi.BILLING_TYPE_PREPAID && ec.BillingCycle != nil { + if ec.BillingCycle.GetMonths() >= 1 && ec.BillingCycle.GetMonths() <= 9 { + params["Period"] = strconv.Itoa(ec.BillingCycle.GetMonths()) + } else if ec.BillingCycle.GetMonths() == 12 || ec.BillingCycle.GetMonths() == 24 || ec.BillingCycle.GetMonths() == 36 { + params["Period"] = strconv.Itoa(ec.BillingCycle.GetMonths()) } else { return nil, fmt.Errorf("region.CreateIElasticcaches invalid billing cycle.reqired month (1~9) or year(1~3)") } diff --git a/pkg/multicloud/hcs/bucket.go b/pkg/multicloud/hcs/bucket.go new file mode 100644 index 0000000000..b35c5c56d4 --- /dev/null +++ b/pkg/multicloud/hcs/bucket.go @@ -0,0 +1,803 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + "yunion.io/x/s3cli" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SBucket struct { + multicloud.SBaseBucket + multicloud.HuaweiTags + + region *SRegion + + Name string + Location string + CreationDate time.Time +} + +func (b *SBucket) GetProjectId() string { + resp, err := b.region.HeadBucket(b.Name) + if err != nil { + return "" + } + epid, _ := resp.ResponseHeaders["epid"] + if len(epid) > 0 { + return epid[0] + } + return "" +} + +func (b *SBucket) GetGlobalId() string { + return b.Name +} + +func (b *SBucket) GetName() string { + return b.Name +} + +func (b *SBucket) GetLocation() string { + return b.Location +} + +func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion { + return b.region +} + +func (b *SBucket) GetCreateAt() time.Time { + return b.CreationDate +} + +func (b *SBucket) GetStorageClass() string { + obscli, err := b.region.getOBSClient() + if err != nil { + log.Errorf("b.region.getOBSClient error %s", err) + return "" + } + output, err := obscli.GetBucketStoragePolicy(b.Name) + if err != nil { + log.Errorf("obscli.GetBucketStoragePolicy error %s", err) + return "" + } + return output.StorageClass +} + +func obsAcl2CannedAcl(acls []obs.Grant) cloudprovider.TBucketACLType { + switch { + case len(acls) == 1: + if acls[0].Grantee.URI == "" && acls[0].Permission == s3cli.PERMISSION_FULL_CONTROL { + return cloudprovider.ACLPrivate + } + case len(acls) == 2: + for _, g := range acls { + if g.Grantee.URI == s3cli.GRANTEE_GROUP_URI_AUTH_USERS && g.Permission == s3cli.PERMISSION_READ { + return cloudprovider.ACLAuthRead + } + if g.Grantee.URI == s3cli.GRANTEE_GROUP_URI_ALL_USERS && g.Permission == s3cli.PERMISSION_READ { + return cloudprovider.ACLPublicRead + } + } + case len(acls) == 3: + for _, g := range acls { + if g.Grantee.URI == s3cli.GRANTEE_GROUP_URI_ALL_USERS && g.Permission == s3cli.PERMISSION_WRITE { + return cloudprovider.ACLPublicReadWrite + } + } + } + return cloudprovider.ACLUnknown +} + +func (b *SBucket) GetAcl() cloudprovider.TBucketACLType { + acl := cloudprovider.ACLPrivate + obscli, err := b.region.getOBSClient() + if err != nil { + log.Errorf("b.region.getOBSClient error %s", err) + return acl + } + output, err := obscli.GetBucketAcl(b.Name) + if err != nil { + log.Errorf("obscli.GetBucketAcl error %s", err) + return acl + } + acl = obsAcl2CannedAcl(output.Grants) + return acl +} + +func (b *SBucket) SetAcl(acl cloudprovider.TBucketACLType) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "b.region.getOBSClient") + } + input := &obs.SetBucketAclInput{} + input.Bucket = b.Name + input.ACL = obs.AclType(string(acl)) + _, err = obscli.SetBucketAcl(input) + if err != nil { + return errors.Wrap(err, "obscli.SetBucketAcl") + } + return nil +} + +func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { + return []cloudprovider.SBucketAccessUrl{ + { + Url: fmt.Sprintf("https://%s.%s", b.Name, b.region.getOBSEndpoint()), + Description: "bucket url", + Primary: true, + }, + { + Url: fmt.Sprintf("https://%s/%s", b.region.getOBSEndpoint(), b.Name), + Description: "obs url", + }, + } +} + +func (b *SBucket) GetStats() cloudprovider.SBucketStats { + stats := cloudprovider.SBucketStats{} + obscli, err := b.region.getOBSClient() + if err != nil { + log.Errorf("b.region.getOBSClient error %s", err) + stats.SizeBytes = -1 + stats.ObjectCount = -1 + return stats + } + output, err := obscli.GetBucketStorageInfo(b.Name) + if err != nil { + log.Errorf("obscli.GetBucketStorageInfo error %s", err) + stats.SizeBytes = -1 + stats.ObjectCount = -1 + return stats + } + stats.SizeBytes = output.Size + stats.ObjectCount = output.ObjectNumber + return stats +} + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + obscli, err := b.region.getOBSClient() + if err != nil { + return result, errors.Wrap(err, "GetOBSClient") + } + input := &obs.ListObjectsInput{} + input.Bucket = b.Name + if len(prefix) > 0 { + input.Prefix = prefix + } + if len(marker) > 0 { + input.Marker = marker + } + if len(delimiter) > 0 { + input.Delimiter = delimiter + } + if maxCount > 0 { + input.MaxKeys = maxCount + } + oResult, err := obscli.ListObjects(input) + if err != nil { + return result, errors.Wrap(err, "ListObjects") + } + result.Objects = make([]cloudprovider.ICloudObject, 0) + for _, object := range oResult.Contents { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: string(object.StorageClass), + Key: object.Key, + SizeBytes: object.Size, + ETag: object.ETag, + LastModified: object.LastModified, + }, + } + result.Objects = append(result.Objects, obj) + } + if oResult.CommonPrefixes != nil { + result.CommonPrefixes = make([]cloudprovider.ICloudObject, 0) + for _, commonPrefix := range oResult.CommonPrefixes { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: commonPrefix, + }, + } + result.CommonPrefixes = append(result.CommonPrefixes, obj) + } + } + result.IsTruncated = oResult.IsTruncated + result.NextMarker = oResult.NextMarker + return result, nil +} + +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.PutObjectInput{} + input.Bucket = b.Name + input.Key = key + input.Body = reader + + if sizeBytes > 0 { + input.ContentLength = sizeBytes + } + if len(storageClassStr) > 0 { + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return err + } + } + if len(cannedAcl) == 0 { + cannedAcl = b.GetAcl() + } + input.ACL = obs.AclType(string(cannedAcl)) + if meta != nil { + val := meta.Get(cloudprovider.META_HEADER_CONTENT_TYPE) + if len(val) > 0 { + input.ContentType = val + } + val = meta.Get(cloudprovider.META_HEADER_CONTENT_MD5) + if len(val) > 0 { + input.ContentMD5 = val + } + extraMeta := make(map[string]string) + for k, v := range meta { + if utils.IsInStringArray(k, []string{ + cloudprovider.META_HEADER_CONTENT_TYPE, + cloudprovider.META_HEADER_CONTENT_MD5, + }) { + continue + } + if len(v[0]) > 0 { + extraMeta[k] = v[0] + } + } + input.Metadata = extraMeta + } + _, err = obscli.PutObject(input) + if err != nil { + return errors.Wrap(err, "PutObject") + } + return nil +} + +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + + input := &obs.InitiateMultipartUploadInput{} + input.Bucket = b.Name + input.Key = key + if meta != nil { + val := meta.Get(cloudprovider.META_HEADER_CONTENT_TYPE) + if len(val) > 0 { + input.ContentType = val + } + extraMeta := make(map[string]string) + for k, v := range meta { + if utils.IsInStringArray(k, []string{ + cloudprovider.META_HEADER_CONTENT_TYPE, + }) { + continue + } + if len(v[0]) > 0 { + extraMeta[k] = v[0] + } + } + input.Metadata = extraMeta + } + if len(cannedAcl) == 0 { + cannedAcl = b.GetAcl() + } + input.ACL = obs.AclType(string(cannedAcl)) + if len(storageClassStr) > 0 { + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return "", errors.Wrap(err, "str2StorageClass") + } + } + output, err := obscli.InitiateMultipartUpload(input) + if err != nil { + return "", errors.Wrap(err, "InitiateMultipartUpload") + } + + return output.UploadId, nil +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, part io.Reader, partSize int64, offset, totalSize int64) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + + input := &obs.UploadPartInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + input.PartNumber = partIndex + input.PartSize = partSize + input.Body = part + output, err := obscli.UploadPart(input) + if err != nil { + return "", errors.Wrap(err, "UploadPart") + } + + return output.ETag, nil +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.CompleteMultipartUploadInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + parts := make([]obs.Part, len(partEtags)) + for i := range partEtags { + parts[i] = obs.Part{ + PartNumber: i + 1, + ETag: partEtags[i], + } + } + input.Parts = parts + _, err = obscli.CompleteMultipartUpload(input) + if err != nil { + return errors.Wrap(err, "CompleteMultipartUpload") + } + + return nil +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + input := &obs.AbortMultipartUploadInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + + _, err = obscli.AbortMultipartUpload(input) + if err != nil { + return errors.Wrap(err, "AbortMultipartUpload") + } + + return nil +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.DeleteObjectInput{} + input.Bucket = b.Name + input.Key = key + _, err = obscli.DeleteObject(input) + if err != nil { + return errors.Wrap(err, "DeleteObject") + } + return nil +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + input := obs.CreateSignedUrlInput{} + input.Bucket = b.Name + input.Key = key + input.Expires = int(expire / time.Second) + switch method { + case "GET": + input.Method = obs.HttpMethodGet + case "PUT": + input.Method = obs.HttpMethodPut + case "DELETE": + input.Method = obs.HttpMethodDelete + default: + return "", errors.Error("unsupported method") + } + output, err := obscli.CreateSignedUrl(&input) + return output.SignedUrl, nil +} + +func (b *SBucket) LimitSupport() cloudprovider.SBucketStats { + return cloudprovider.SBucketStats{ + SizeBytes: 1, + ObjectCount: -1, + } +} + +func (b *SBucket) GetLimit() cloudprovider.SBucketStats { + stats := cloudprovider.SBucketStats{} + obscli, err := b.region.getOBSClient() + if err != nil { + log.Errorf("getOBSClient error %s", err) + return stats + } + output, err := obscli.GetBucketQuota(b.Name) + if err != nil { + return stats + } + stats.SizeBytes = output.Quota + return stats +} + +func (b *SBucket) SetLimit(limit cloudprovider.SBucketStats) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "getOBSClient") + } + input := &obs.SetBucketQuotaInput{} + input.Bucket = b.Name + input.Quota = limit.SizeBytes + _, err = obscli.SetBucketQuota(input) + if err != nil { + return errors.Wrap(err, "SetBucketQuota") + } + return nil +} + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.CopyObjectInput{} + input.Bucket = b.Name + input.Key = destKey + input.CopySourceBucket = srcBucket + input.CopySourceKey = srcKey + if len(storageClassStr) > 0 { + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return err + } + } + if len(cannedAcl) == 0 { + cannedAcl = b.GetAcl() + } + input.ACL = obs.AclType(string(cannedAcl)) + if meta != nil { + val := meta.Get(cloudprovider.META_HEADER_CONTENT_TYPE) + if len(val) > 0 { + input.ContentType = val + } + extraMeta := make(map[string]string) + for k, v := range meta { + if utils.IsInStringArray(k, []string{ + cloudprovider.META_HEADER_CONTENT_TYPE, + }) { + continue + } + if len(v[0]) > 0 { + extraMeta[k] = v[0] + } + } + input.Metadata = extraMeta + input.MetadataDirective = obs.ReplaceMetadata + } else { + input.MetadataDirective = obs.CopyMetadata + } + _, err = obscli.CopyObject(input) + if err != nil { + return errors.Wrap(err, "obscli.CopyObject") + } + return nil +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "GetOBSClient") + } + input := &obs.GetObjectInput{} + input.Bucket = b.Name + input.Key = key + if rangeOpt != nil { + input.RangeStart = rangeOpt.Start + input.RangeEnd = rangeOpt.End + } + output, err := obscli.GetObject(input) + if err != nil { + return nil, errors.Wrap(err, "obscli.GetObject") + } + return output.Body, nil +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + input := &obs.CopyPartInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + input.PartNumber = partIndex + input.CopySourceBucket = srcBucket + input.CopySourceKey = srcKey + input.CopySourceRangeStart = srcOffset + input.CopySourceRangeEnd = srcOffset + srcLength - 1 + output, err := obscli.CopyPart(input) + if err != nil { + return "", errors.Wrap(err, "CopyPart") + } + return output.ETag, nil +} + +func (b *SBucket) SetWebsite(websitConf cloudprovider.SBucketWebsiteConf) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + obsWebConf := obs.SetBucketWebsiteConfigurationInput{} + obsWebConf.Bucket = b.Name + obsWebConf.BucketWebsiteConfiguration = obs.BucketWebsiteConfiguration{ + IndexDocument: obs.IndexDocument{Suffix: websitConf.Index}, + ErrorDocument: obs.ErrorDocument{Key: websitConf.ErrorDocument}, + } + _, err = obscli.SetBucketWebsiteConfiguration(&obsWebConf) + if err != nil { + return errors.Wrap(err, "obscli.SetBucketWebsiteConfiguration(&obsWebConf)") + } + return nil +} + +func (b *SBucket) GetWebsiteConf() (cloudprovider.SBucketWebsiteConf, error) { + result := cloudprovider.SBucketWebsiteConf{} + obscli, err := b.region.getOBSClient() + if err != nil { + return result, errors.Wrap(err, "GetOBSClient") + } + out, err := obscli.GetBucketWebsiteConfiguration(b.Name) + if out == nil { + return result, nil + } + result.Index = out.IndexDocument.Suffix + result.ErrorDocument = out.ErrorDocument.Key + result.Url = fmt.Sprintf("https://%s.obs-website.%s.myhuaweicloud.com", b.Name, b.region.GetId()) + return result, nil +} + +func (b *SBucket) DeleteWebSiteConf() error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + _, err = obscli.DeleteBucketWebsiteConfiguration(b.Name) + if err != nil { + return errors.Wrapf(err, "obscli.DeleteBucketWebsiteConfiguration(%s)", b.Name) + } + return nil +} + +func (b *SBucket) SetCORS(rules []cloudprovider.SBucketCORSRule) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + opts := []obs.CorsRule{} + for i := range rules { + opts = append(opts, obs.CorsRule{ + AllowedOrigin: rules[i].AllowedOrigins, + AllowedMethod: rules[i].AllowedMethods, + AllowedHeader: rules[i].AllowedHeaders, + MaxAgeSeconds: rules[i].MaxAgeSeconds, + ExposeHeader: rules[i].ExposeHeaders, + }) + } + + input := obs.SetBucketCorsInput{} + input.Bucket = b.Name + input.BucketCors.CorsRules = opts + _, err = obscli.SetBucketCors(&input) + if err != nil { + return errors.Wrapf(err, "obscli.SetBucketCors(%s)", jsonutils.Marshal(input).String()) + } + return nil +} + +func (b *SBucket) GetCORSRules() ([]cloudprovider.SBucketCORSRule, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "GetOBSClient") + } + conf, err := obscli.GetBucketCors(b.Name) + if err != nil { + if !strings.Contains(err.Error(), "NoSuchCORSConfiguration") { + return nil, errors.Wrapf(err, "obscli.GetBucketCors(%s)", b.Name) + } + } + if conf == nil { + return nil, nil + } + result := []cloudprovider.SBucketCORSRule{} + for i := range conf.CorsRules { + result = append(result, cloudprovider.SBucketCORSRule{ + AllowedOrigins: conf.CorsRules[i].AllowedOrigin, + AllowedMethods: conf.CorsRules[i].AllowedMethod, + AllowedHeaders: conf.CorsRules[i].AllowedHeader, + MaxAgeSeconds: conf.CorsRules[i].MaxAgeSeconds, + ExposeHeaders: conf.CorsRules[i].ExposeHeader, + Id: strconv.Itoa(i), + }) + } + return result, nil +} + +func (b *SBucket) DeleteCORS() error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + _, err = obscli.DeleteBucketCors(b.Name) + if err != nil { + return errors.Wrapf(err, "obscli.DeleteBucketCors(%s)", b.Name) + } + return nil +} + +func (b *SBucket) GetTags() (map[string]string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "GetOBSClient") + } + tagresult, err := obscli.GetBucketTagging(b.Name) + if err != nil { + if strings.Contains(err.Error(), "404") { + return nil, nil + } + return nil, errors.Wrapf(err, "osscli.GetBucketTagging(%s)", b.Name) + } + result := map[string]string{} + for i := range tagresult.Tags { + result[tagresult.Tags[i].Key] = tagresult.Tags[i].Value + } + return result, nil +} + +func (b *SBucket) SetTags(tags map[string]string, replace bool) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + _, err = obscli.DeleteBucketTagging(b.Name) + if err != nil { + return errors.Wrapf(err, "DeleteBucketTagging") + } + + if len(tags) == 0 { + return nil + } + + input := obs.SetBucketTaggingInput{BucketTagging: obs.BucketTagging{}} + input.Bucket = b.Name + for k, v := range tags { + input.BucketTagging.Tags = append(input.BucketTagging.Tags, obs.Tag{Key: k, Value: v}) + } + + _, err = obscli.SetBucketTagging(&input) + if err != nil { + return errors.Wrapf(err, "obscli.SetBucketTagging(%s)", jsonutils.Marshal(input).String()) + } + return nil +} + +func (b *SBucket) ListMultipartUploads() ([]cloudprovider.SBucketMultipartUploads, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "GetOBSClient") + } + result := []cloudprovider.SBucketMultipartUploads{} + + input := obs.ListMultipartUploadsInput{Bucket: b.Name} + keyMarker := "" + uploadIDMarker := "" + for { + if len(keyMarker) > 0 { + input.KeyMarker = keyMarker + } + if len(uploadIDMarker) > 0 { + input.UploadIdMarker = uploadIDMarker + } + + output, err := obscli.ListMultipartUploads(&input) + if err != nil { + return nil, errors.Wrap(err, " coscli.Bucket.ListMultipartUploads(context.Background(), &input)") + } + for i := range output.Uploads { + temp := cloudprovider.SBucketMultipartUploads{ + ObjectName: output.Uploads[i].Key, + UploadID: output.Uploads[i].UploadId, + Initiator: output.Uploads[i].Initiator.DisplayName, + Initiated: output.Uploads[i].Initiated, + } + result = append(result, temp) + } + keyMarker = output.NextKeyMarker + uploadIDMarker = output.NextUploadIdMarker + if !output.IsTruncated { + break + } + } + + return result, nil +} + +func (self *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { + return self.CreateBucket(name, storageClassStr, aclStr) +} + +func (self *SRegion) CreateBucket(name string, storageClassStr string, aclStr string) error { + obsClient, err := self.getOBSClient() + if err != nil { + return errors.Wrap(err, "self.getOBSClient") + } + input := &obs.CreateBucketInput{} + input.Bucket = name + input.Location = self.GetId() + if len(aclStr) > 0 { + if strings.EqualFold(aclStr, string(obs.AclPrivate)) { + input.ACL = obs.AclPrivate + } else if strings.EqualFold(aclStr, string(obs.AclPublicRead)) { + input.ACL = obs.AclPublicRead + } else if strings.EqualFold(aclStr, string(obs.AclPublicReadWrite)) { + input.ACL = obs.AclPublicReadWrite + } else { + return errors.Error("unsupported acl") + } + } + if len(storageClassStr) > 0 { + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return err + } + } + _, err = obsClient.CreateBucket(input) + if err != nil { + return errors.Wrap(err, "obsClient.CreateBucket") + } + return nil +} diff --git a/pkg/multicloud/hcs/dbinstance.go b/pkg/multicloud/hcs/dbinstance.go new file mode 100644 index 0000000000..db781c9119 --- /dev/null +++ b/pkg/multicloud/hcs/dbinstance.go @@ -0,0 +1,706 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + billing_api "yunion.io/x/onecloud/pkg/apis/billing" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +type SBackupStrategy struct { + KeepDays int + StartTime string +} + +type SDatastore struct { + Type string + Version string +} + +type SHa struct { + ReplicationMode string +} + +type SNonde struct { + AvailabilityZone string + Id string + Name string + Role string + Staus string +} + +type SVolume struct { + Size int + Type string +} + +type SRelatedInstance struct { + Id string + Type string +} + +type SDBInstance struct { + multicloud.SDBInstanceBase + multicloud.HuaweiTags + region *SRegion + + flavorCache []SDBInstanceFlavor + + BackupStrategy SBackupStrategy + Created string //time.Time + Datastore SDatastore + DbUserName string + DIskEncryptionId string + FlavorRef string + Ha SHa + Id string + MaintenanceWindow string + Name string + Nodes []SNonde + Port int + PrivateIps []string + PublicIps []string + Region string + RelatedInstance []SRelatedInstance + SecurityGroupId string + Status string + SubnetId string + SwitchStrategy string + TimeZone string + Type string + Updated string //time.Time + Volume SVolume + VpcId string + EnterpriseProjectId string +} + +func (region *SRegion) GetDBInstances() ([]SDBInstance, error) { + dbinstances := []SDBInstance{} + err := region.rdsList("instances", nil, nil) + return dbinstances, err +} + +func (region *SRegion) GetDBInstance(instanceId string) (*SDBInstance, error) { + if len(instanceId) == 0 { + return nil, cloudprovider.ErrNotFound + } + instance := SDBInstance{region: region} + res := &SDBInstance{} + err := region.rdsGet(fmt.Sprintf("instance/%s", instanceId), res) + return &instance, err +} + +func (rds *SDBInstance) GetName() string { + return rds.Name +} + +func (rds *SDBInstance) GetId() string { + return rds.Id +} + +func (rds *SDBInstance) GetGlobalId() string { + return rds.GetId() +} + +// 值为“BUILD”，表示实例正在创建。 +// 值为“ACTIVE”，表示实例正常。 +// 值为“FAILED”，表示实例异常。 +// 值为“FROZEN”，表示实例冻结。 +// 值为“MODIFYING”，表示实例正在扩容。 +// 值为“REBOOTING”，表示实例正在重启。 +// 值为“RESTORING”，表示实例正在恢复。 +// 值为“MODIFYING INSTANCE TYPE”，表示实例正在转主备。 +// 值为“SWITCHOVER”，表示实例正在主备切换。 +// 值为“MIGRATING”，表示实例正在迁移。 +// 值为“BACKING UP”，表示实例正在进行备份。 +// 值为“MODIFYING DATABASE PORT”，表示实例正在修改数据库端口。 +// 值为“STORAGE FULL”，表示实例磁盘空间满。 + +func (rds *SDBInstance) GetStatus() string { + switch rds.Status { + case "BUILD", "MODIFYING", "MODIFYING INSTANCE TYPE", "SWITCHOVER", "MODIFYING DATABASE PORT": + return api.DBINSTANCE_DEPLOYING + case "ACTIVE": + return api.DBINSTANCE_RUNNING + case "FAILED", "FROZEN", "STORAGE FULL": + return api.DBINSTANCE_UNKNOWN + case "REBOOTING": + return api.DBINSTANCE_REBOOTING + case "RESTORING": + return api.DBINSTANCE_RESTORING + case "MIGRATING": + return api.DBINSTANCE_MIGRATING + case "BACKING UP": + return api.DBINSTANCE_BACKING_UP + } + return rds.Status +} + +func (rds *SDBInstance) GetBillingType() string { + return billing_api.BILLING_TYPE_POSTPAID +} + +func (rds *SDBInstance) GetSecurityGroupIds() ([]string, error) { + return []string{rds.SecurityGroupId}, nil +} + +func (rds *SDBInstance) fetchFlavor() error { + if len(rds.flavorCache) > 0 { + return nil + } + flavors, err := rds.region.GetDBInstanceFlavors(rds.Datastore.Type, rds.Datastore.Version) + if err != nil { + return err + } + rds.flavorCache = flavors + return nil +} + +func (rds *SDBInstance) GetExpiredAt() time.Time { + return time.Time{} +} + +func (rds *SDBInstance) GetStorageType() string { + return rds.Volume.Type +} + +func (rds *SDBInstance) GetCreatedAt() time.Time { + t, err := time.Parse("2006-01-02T15:04:05Z0700", rds.Created) + if err != nil { + return time.Time{} + } + return t +} + +func (rds *SDBInstance) GetEngine() string { + return rds.Datastore.Type +} + +func (rds *SDBInstance) GetEngineVersion() string { + return rds.Datastore.Version +} + +func (rds *SDBInstance) GetInstanceType() string { + return rds.FlavorRef +} + +func (rds *SDBInstance) GetCategory() string { + switch rds.Type { + case "Single": + return api.HUAWEI_DBINSTANCE_CATEGORY_SINGLE + case "Ha": + return api.HUAWEI_DBINSTANCE_CATEGORY_HA + case "Replica": + return api.HUAWEI_DBINSTANCE_CATEGORY_REPLICA + } + return rds.Type +} + +func (rds *SDBInstance) GetVcpuCount() int { + err := rds.fetchFlavor() + if err != nil { + log.Errorf("failed to fetch flavors: %v", err) + return 0 + } + for _, flavor := range rds.flavorCache { + if flavor.SpecCode == rds.FlavorRef { + return flavor.Vcpus + } + } + return 0 +} + +func (rds *SDBInstance) GetVmemSizeMB() int { + err := rds.fetchFlavor() + if err != nil { + log.Errorf("failed to fetch flavors: %v", err) + return 0 + } + for _, flavor := range rds.flavorCache { + if flavor.SpecCode == rds.FlavorRef { + return flavor.Ram * 1024 + } + } + return 0 +} + +func (rds *SDBInstance) GetDiskSizeGB() int { + return rds.Volume.Size +} + +func (rds *SDBInstance) GetPort() int { + return rds.Port +} + +func (rds *SDBInstance) GetMaintainTime() string { + return rds.MaintenanceWindow +} + +func (rds *SDBInstance) GetIVpcId() string { + return rds.VpcId +} + +func (rds *SDBInstance) GetProjectId() string { + return rds.EnterpriseProjectId +} + +func (rds *SDBInstance) Refresh() error { + instance, err := rds.region.GetDBInstance(rds.Id) + if err != nil { + return err + } + return jsonutils.Update(rds, instance) +} + +func (rds *SDBInstance) GetZone1Id() string { + return rds.GetZoneIdByRole("master") +} + +func (rds *SDBInstance) GetZoneIdByRole(role string) string { + // for _, node := range rds.Nodes { + // if node.Role == role { + // zone, err := rds.region.getZoneById(node.AvailabilityZone) + // if err != nil { + // log.Errorf("failed to found zone %s for rds %s error: %v", node.AvailabilityZone, rds.Name, err) + // return "" + // } + // return zone.GetGlobalId() + // } + // } + return "" +} + +func (rds *SDBInstance) GetZone2Id() string { + return rds.GetZoneIdByRole("slave") +} + +func (rds *SDBInstance) GetZone3Id() string { + return "" +} + +type SRdsNetwork struct { + SubnetId string + IP string +} + +func (rds *SDBInstance) GetDBNetworks() ([]cloudprovider.SDBInstanceNetwork, error) { + ret := []cloudprovider.SDBInstanceNetwork{} + for _, ip := range rds.PrivateIps { + network := cloudprovider.SDBInstanceNetwork{ + IP: ip, + NetworkId: rds.SubnetId, + } + ret = append(ret, network) + } + return ret, nil +} + +func (rds *SDBInstance) GetInternalConnectionStr() string { + for _, ip := range rds.PrivateIps { + return ip + } + return "" +} + +func (rds *SDBInstance) GetConnectionStr() string { + for _, ip := range rds.PublicIps { + return ip + } + return "" +} + +func (region *SRegion) GetIDBInstanceById(instanceId string) (cloudprovider.ICloudDBInstance, error) { + dbinstance, err := region.GetDBInstance(instanceId) + if err != nil { + log.Errorf("failed to get dbinstance by id %s error: %v", instanceId, err) + } + return dbinstance, err +} + +func (region *SRegion) GetIDBInstances() ([]cloudprovider.ICloudDBInstance, error) { + instances, err := region.GetDBInstances() + if err != nil { + return nil, errors.Wrapf(err, "region.GetDBInstances()") + } + idbinstances := []cloudprovider.ICloudDBInstance{} + for i := 0; i < len(instances); i++ { + instances[i].region = region + idbinstances = append(idbinstances, &instances[i]) + } + return idbinstances, nil +} + +func (rds *SDBInstance) GetIDBInstanceParameters() ([]cloudprovider.ICloudDBInstanceParameter, error) { + parameters, err := rds.region.GetDBInstanceParameters(rds.Id) + if err != nil { + return nil, err + } + iparameters := []cloudprovider.ICloudDBInstanceParameter{} + for i := 0; i < len(parameters); i++ { + iparameters = append(iparameters, ¶meters[i]) + } + return iparameters, nil +} + +func (rds *SDBInstance) GetIDBInstanceDatabases() ([]cloudprovider.ICloudDBInstanceDatabase, error) { + databases, err := rds.region.GetDBInstanceDatabases(rds.Id) + if err != nil { + return nil, errors.Wrap(err, "rds.region.GetDBInstanceDatabases(rds.Id)") + } + + idatabase := []cloudprovider.ICloudDBInstanceDatabase{} + for i := 0; i < len(databases); i++ { + databases[i].instance = rds + idatabase = append(idatabase, &databases[i]) + } + return idatabase, nil +} + +func (rds *SDBInstance) GetIDBInstanceAccounts() ([]cloudprovider.ICloudDBInstanceAccount, error) { + accounts, err := rds.region.GetDBInstanceAccounts(rds.Id) + if err != nil { + return nil, errors.Wrap(err, "rds.region.GetDBInstanceAccounts(rds.Id)") + } + + user := "root" + if rds.GetEngine() == api.DBINSTANCE_TYPE_SQLSERVER { + user = "rduser" + } + + accounts = append(accounts, SDBInstanceAccount{ + Name: user, + instance: rds, + }) + + iaccounts := []cloudprovider.ICloudDBInstanceAccount{} + for i := 0; i < len(accounts); i++ { + accounts[i].instance = rds + iaccounts = append(iaccounts, &accounts[i]) + } + return iaccounts, nil +} + +func (rds *SDBInstance) Delete() error { + return rds.region.DeleteDBInstance(rds.Id) +} + +func (region *SRegion) DeleteDBInstance(instanceId string) error { + err := region.client.rdsDelete(region.Id, instanceId) + return err +} + +func (region *SRegion) CreateIDBInstance(desc *cloudprovider.SManagedDBInstanceCreateConfig) (cloudprovider.ICloudDBInstance, error) { + zoneIds := []string{} + zones, err := region.GetIZones() + if err != nil { + return nil, err + } + for _, zone := range zones { + zoneIds = append(zoneIds, zone.GetId()) + } + + if len(desc.SecgroupIds) == 0 { + return nil, fmt.Errorf("Missing secgroupId") + } + + params := map[string]interface{}{ + "region": region.Id, + "name": desc.Name, + "datastore": map[string]string{ + "type": desc.Engine, + "version": desc.EngineVersion, + }, + "password": desc.Password, + "volume": map[string]interface{}{ + "type": desc.StorageType, + "size": desc.DiskSizeGB, + }, + "vpc_id": desc.VpcId, + "subnet_id": desc.NetworkId, + "security_group_id": desc.SecgroupIds[0], + } + + if len(desc.ProjectId) > 0 { + params["enterprise_project_id"] = desc.ProjectId + } + + if len(desc.MasterInstanceId) > 0 { + params["replica_of_id"] = desc.MasterInstanceId + delete(params, "security_group_id") + } + + if len(desc.RdsId) > 0 && len(desc.BackupId) > 0 { + params["restore_point"] = map[string]interface{}{ + "backup_id": desc.BackupId, + "instance_id": desc.RdsId, + "type": "backup", + } + } + + switch desc.Category { + case api.HUAWEI_DBINSTANCE_CATEGORY_HA: + switch desc.Engine { + case api.DBINSTANCE_TYPE_MYSQL, api.DBINSTANCE_TYPE_POSTGRESQL: + params["ha"] = map[string]string{ + "mode": "Ha", + "replication_mode": "async", + } + case api.DBINSTANCE_TYPE_SQLSERVER: + params["ha"] = map[string]string{ + "mode": "Ha", + "replication_mode": "sync", + } + } + case api.HUAWEI_DBINSTANCE_CATEGORY_SINGLE: + case api.HUAWEI_DBINSTANCE_CATEGORY_REPLICA: + } + + if desc.BillingCycle != nil { + periodType := "month" + periodNum := desc.BillingCycle.GetMonths() + if desc.BillingCycle.GetYears() > 0 { + periodType = "year" + periodNum = desc.BillingCycle.GetYears() + } + params["charge_info"] = map[string]interface{}{ + "charge_mode": "prePaid", + "period_type": periodType, + "period_num": periodNum, + "is_auto_renew": false, + } + } + params["flavor_ref"] = desc.InstanceType + params["availability_zone"] = desc.ZoneId + resp, err := region.client.request(httputils.POST, region.client._url("instance", "v3", region.Id, "instance"), nil, params) + if err != nil { + return nil, errors.Wrapf(err, "Create") + } + + instance := &SDBInstance{region: region} + err = resp.Unmarshal(instance, "instance") + if err != nil { + return nil, errors.Wrap(err, `resp.Unmarshal(&instance, "instance")`) + } + if jobId, _ := resp.GetString("job_id"); len(jobId) > 0 { + err = cloudprovider.Wait(10*time.Second, 20*time.Minute, func() (bool, error) { + + job, err := region.client.request(httputils.POST, region.client._url(fmt.Sprintf("instance/%s", jobId), "v3", region.Id, "instance"), nil, params) + if err != nil { + return false, nil + } + status, _ := job.GetString("status") + process, _ := job.GetString("process") + log.Debugf("create dbinstance job %s status: %s process: %s", jobId, status, process) + if status == "Completed" { + return true, nil + } + if status == "Failed" { + return false, fmt.Errorf("create failed") + } + return false, nil + }) + } + return instance, err +} + +func (rds *SDBInstance) Reboot() error { + // return rds.region.RebootDBInstance(rds.Id) + return nil +} + +func (rds *SDBInstance) OpenPublicConnection() error { + return fmt.Errorf("Huawei current not support this operation") + //return rds.region.PublicConnectionAction(rds.Id, "openRC") +} + +func (rds *SDBInstance) ClosePublicConnection() error { + return fmt.Errorf("Huawei current not support this operation") + //return rds.region.PublicConnectionAction(rds.Id, "closeRC") +} + +func (region *SRegion) PublicConnectionAction(instanceId string, action string) error { + return cloudprovider.ErrNotImplemented + /*resp, err := region.ecsClient.DBInstance.PerformAction2(action, instanceId, nil, "") + if err != nil { + return errors.Wrapf(err, "rds.%s", action) + } + + if jobId, _ := resp.GetString("job_id"); len(jobId) > 0 { + err = cloudprovider.WaitCreated(10*time.Second, 20*time.Minute, func() bool { + job := struct { + Status string + Process string + }{} + query := url.Values{} + query.Add("id", jobId) + err := region.rdsJobGet("jobs", query, job) + if err != nil { + log.Errorf("failed to get job %s info error: %v", jobId, err) + return false + } + if job.Status == "Completed" { + return true + } + log.Debugf("%s dbinstance job %s status: %s process: %s", action, jobId, job.Status, job.Process) + return false + }) + } + + return nil + */ +} + +func (region *SRegion) RebootDBInstance(instanceId string) error { + return cloudprovider.ErrNotImplemented + /*params := jsonutils.Marshal(map[string]interface{}{ + "restart": map[string]string{}, + }) + resp, err := region.ecsClient.DBInstance.PerformAction2("action", instanceId, params, "") + if err != nil { + return err + } + if jobId, _ := resp.GetString("job_id"); len(jobId) > 0 { + err = cloudprovider.WaitCreated(10*time.Second, 20*time.Minute, func() bool { + job := struct { + Status string + Process string + }{} + query := url.Values{} + query.Add("id", jobId) + err := region.rdsJobGet("jobs", query, job) + if err != nil { + log.Errorf("failed to get job %s info error: %v", jobId, err) + return false + } + if job.Status == "Completed" { + return true + } + log.Debugf("%s dbinstance job %s status: %s process: %s", action, jobId, job.Status, job.Process) + return false + }) + } + return err*/ +} + +func (rds *SDBInstance) CreateAccount(conf *cloudprovider.SDBInstanceAccountCreateConfig) error { + return rds.region.CreateDBInstanceAccount(rds.Id, conf.Name, conf.Password) +} + +func (region *SRegion) CreateDBInstanceAccount(instanceId, account, password string) error { + params := map[string]interface{}{ + "name": account, + "password": password, + } + + err := region.rdsCreate(fmt.Sprintf("instances/%s/db_user", instanceId), params, nil) + return err +} + +func (rds *SDBInstance) CreateDatabase(conf *cloudprovider.SDBInstanceDatabaseCreateConfig) error { + return rds.region.CreateDBInstanceDatabase(rds.Id, conf.Name, conf.CharacterSet) +} + +func (region *SRegion) CreateDBInstanceDatabase(instanceId, database, characterSet string) error { + params := map[string]interface{}{ + "name": database, + "character_set": characterSet, + } + err := region.rdsCreate(fmt.Sprintf("instances/%s/database", instanceId), params, nil) + return err +} + +func (rds *SDBInstance) ChangeConfig(cxt context.Context, desc *cloudprovider.SManagedDBInstanceChangeConfig) error { + return rds.region.ChangeDBInstanceConfig(rds.Id, desc.InstanceType, desc.DiskSizeGB) +} + +func (region *SRegion) ChangeDBInstanceConfig(instanceId string, instanceType string, diskSizeGb int) error { + instance, err := region.GetIDBInstanceById(instanceId) + if err != nil { + return errors.Wrapf(err, "region.GetIDBInstanceById(%s)", instanceId) + } + + if len(instanceType) > 0 { + params := map[string]interface{}{ + "resize_flavor": map[string]string{ + "spec_code": instanceType, + }, + } + err := region.rdsPerform(fmt.Sprintf("instances/%s", instanceId), "action", params, nil) + if err != nil { + return errors.Wrap(err, "resize_flavor") + } + cloudprovider.WaitStatus(instance, api.DBINSTANCE_RUNNING, time.Second*5, time.Minute*30) + } + if diskSizeGb > 0 { + params := map[string]interface{}{ + "enlarge_volume": map[string]int{ + "size": diskSizeGb, + }, + } + err := region.rdsPerform(fmt.Sprintf("instances/%s", instanceId), "action", params, nil) + if err != nil { + return errors.Wrap(err, "enlarge_volume") + } + cloudprovider.WaitStatus(instance, api.DBINSTANCE_RUNNING, time.Second*5, time.Minute*30) + } + return nil +} + +func (rds *SDBInstance) RecoveryFromBackup(conf *cloudprovider.SDBInstanceRecoveryConfig) error { + if len(conf.OriginDBInstanceExternalId) == 0 { + conf.OriginDBInstanceExternalId = rds.Id + } + return rds.region.RecoveryDBInstanceFromBackup(rds.Id, conf.OriginDBInstanceExternalId, conf.BackupId, conf.Databases) +} + +func (region *SRegion) RecoveryDBInstanceFromBackup(target, origin string, backupId string, databases map[string]string) error { + source := map[string]interface{}{ + "type": "backup", + "backup_id": backupId, + } + if len(origin) > 0 { + source["instance_id"] = origin + } + if len(databases) > 0 { + source["database_name"] = databases + } + params := map[string]interface{}{ + "source": source, + "target": map[string]string{ + "instance_id": target, + }, + } + err := region.rdsPerform("instance", "recovery", params, nil) + if err != nil { + return errors.Wrap(err, "dbinstance.recovery") + } + return nil +} + +func (rds *SDBInstance) Renew(bc billing.SBillingCycle) error { + return cloudprovider.ErrNotSupported +} diff --git a/pkg/multicloud/hcs/dbinstance_account.go b/pkg/multicloud/hcs/dbinstance_account.go new file mode 100644 index 0000000000..9717eeaddd --- /dev/null +++ b/pkg/multicloud/hcs/dbinstance_account.go @@ -0,0 +1,125 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SDBInstanceAccount struct { + multicloud.SDBInstanceAccountBase + multicloud.HuaweiTags + instance *SDBInstance + Name string +} + +func (account *SDBInstanceAccount) GetName() string { + return account.Name +} + +func (account *SDBInstanceAccount) Delete() error { + return account.instance.region.rdsDelete(fmt.Sprintf("%s/db_user/%s", account.instance.Id, account.Name)) +} + +func (account *SDBInstanceAccount) GetIDBInstanceAccountPrivileges() ([]cloudprovider.ICloudDBInstanceAccountPrivilege, error) { + privileges, err := account.instance.region.GetDBInstancePrivileges(account.instance.Id, account.Name) + if err != nil { + return nil, err + } + iprivileves := []cloudprovider.ICloudDBInstanceAccountPrivilege{} + for i := 0; i < len(privileges); i++ { + privileges[i].account = account + iprivileves = append(iprivileves, &privileges[i]) + } + return iprivileves, nil +} + +func (region *SRegion) GetDBInstanceAccounts(instanceId string) ([]SDBInstanceAccount, error) { + accounts := []SDBInstanceAccount{} + err := region.rdsList(fmt.Sprintf("instances/%s/db_user/detail", instanceId), nil, accounts) + if err != nil { + return nil, err + } + return accounts, nil +} + +func (region *SRegion) GetDBInstancePrivileges(instanceId string, username string) ([]SDatabasePrivilege, error) { + query := url.Values{} + query.Add("iser-name", username) + privileges := []SDatabasePrivilege{} + err := region.rdsList(fmt.Sprintf("instances/%s/db_user/database", instanceId), query, privileges) + if err != nil { + return nil, err + } + return privileges, nil +} + +func (account *SDBInstanceAccount) RevokePrivilege(database string) error { + return account.instance.region.RevokeDBInstancePrivilege(account.instance.Id, account.Name, database) +} + +func (region *SRegion) RevokeDBInstancePrivilege(instanceId string, account, database string) error { + params := map[string]interface{}{ + "db_name": database, + "users": []map[string]interface{}{ + map[string]interface{}{ + "name": account, + }, + }, + } + return region.rdsDBPrivilegesDelete(fmt.Sprintf("instances/%s/db_privilege", instanceId), params) +} + +func (account *SDBInstanceAccount) GrantPrivilege(database, privilege string) error { + return account.instance.region.GrantDBInstancePrivilege(account.instance.Id, account.Name, database, privilege) +} + +func (region *SRegion) GrantDBInstancePrivilege(instanceId string, account, database string, privilege string) error { + readonly := false + switch privilege { + case api.DATABASE_PRIVILEGE_R: + readonly = true + case api.DATABASE_PRIVILEGE_RW: + default: + return fmt.Errorf("Unknown privilege %s", privilege) + } + params := map[string]interface{}{ + "db_name": database, + "users": []map[string]interface{}{ + map[string]interface{}{ + "name": account, + "readonly": readonly, + }, + }, + } + resp := &struct { + Resp string + }{} + err := region.rdsDBPrivilegesGrant(fmt.Sprintf("instances/%s/db_privilege", instanceId), params, resp) + return err +} + +func (account *SDBInstanceAccount) ResetPassword(password string) error { + return account.instance.region.ResetDBInstanceAccountPassword(account.instance.Id, account.Name, password) +} + +func (region *SRegion) ResetDBInstanceAccountPassword(instanceId, account, password string) error { + return fmt.Errorf("The API does not exist or has not been published in the environment") +} diff --git a/pkg/multicloud/hcs/dbinstance_backup.go b/pkg/multicloud/hcs/dbinstance_backup.go new file mode 100644 index 0000000000..3afb8d08bc --- /dev/null +++ b/pkg/multicloud/hcs/dbinstance_backup.go @@ -0,0 +1,238 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SDBInstanceBackup struct { + multicloud.SDBInstanceBackupBase + multicloud.HuaweiTags + region *SRegion + + BeginTime string + Datastore SDatastore + EndTime string + Id string + InstanceId string + Name string + Size int + Status string + Type string +} + +func (backup *SDBInstanceBackup) GetId() string { + return backup.Id +} + +func (backup *SDBInstanceBackup) GetGlobalId() string { + return backup.Id +} + +func (backup *SDBInstanceBackup) GetName() string { + return backup.Name +} + +func (backup *SDBInstanceBackup) GetEngine() string { + return backup.Datastore.Type +} + +func (backup *SDBInstanceBackup) GetEngineVersion() string { + return backup.Datastore.Version +} + +func (backup *SDBInstanceBackup) GetStartTime() time.Time { + //2019-08-05T08:00:02+0000 + t, err := time.Parse("2006-01-02T15:04:05Z0700", backup.BeginTime) + if err != nil { + return time.Time{} + } + return t +} + +func (backup *SDBInstanceBackup) GetEndTime() time.Time { + t, err := time.Parse("2006-01-02T15:04:05Z0700", backup.EndTime) + if err != nil { + return time.Time{} + } + return t +} + +func (backup *SDBInstanceBackup) GetBackupMode() string { + switch backup.Type { + case "manual": + return api.BACKUP_MODE_MANUAL + default: + return api.BACKUP_MODE_AUTOMATED + } +} + +func (backup *SDBInstanceBackup) GetStatus() string { + switch backup.Status { + case "COMPLETED": + return api.DBINSTANCE_BACKUP_READY + case "FAILED": + return api.DBINSTANCE_BACKUP_FAILED + case "BUILDING": + return api.DBINSTANCE_BACKUP_CREATING + case "DELETING": + return api.DBINSTANCE_BACKUP_DELETING + default: + return api.DBINSTANCE_BACKUP_UNKNOWN + } +} + +func (backup *SDBInstanceBackup) GetBackupSizeMb() int { + return backup.Size / 1024 +} + +func (backup *SDBInstanceBackup) GetDBNames() string { + return "" +} + +func (backup *SDBInstanceBackup) Delete() error { + return backup.region.DeleteDBInstanceBackup(backup.Id) +} + +func (region *SRegion) DeleteDBInstanceBackup(backupId string) error { + err := region.rdsDelete(fmt.Sprintf("backups/%s", backupId)) + return err +} + +func (backup *SDBInstanceBackup) GetDBInstanceId() string { + return backup.InstanceId +} + +func (region *SRegion) GetDBInstanceBackups(instanceId, backupId string) ([]SDBInstanceBackup, error) { + query := url.Values{} + query.Add("instance_id", instanceId) + if len(backupId) > 0 { + query.Add("backup_id", backupId) + } + backups := []SDBInstanceBackup{} + err := region.rdsList("offsite-backups", query, backups) + if err != nil { + return nil, err + } + return backups, nil +} + +func (region *SRegion) GetIDBInstanceBackups() ([]cloudprovider.ICloudDBInstanceBackup, error) { + dbinstnaces, err := region.GetIDBInstances() + if err != nil { + return nil, err + } + ibackups := []cloudprovider.ICloudDBInstanceBackup{} + for i := 0; i < len(dbinstnaces); i++ { + _dbinstance := dbinstnaces[i].(*SDBInstance) + _ibackup, err := _dbinstance.GetIDBInstanceBackups() + if err != nil { + return nil, errors.Wrapf(err, "_dbinstance(%v).GetIDBInstanceBackups", _dbinstance) + } + ibackups = append(ibackups, _ibackup...) + } + return ibackups, nil +} + +func (region *SRegion) GetIDBInstanceBackupById(backupId string) (cloudprovider.ICloudDBInstanceBackup, error) { + backups, err := region.GetIDBInstanceBackups() + if err != nil { + return nil, errors.Wrap(err, "region.GetIDBInstanceBackups") + } + for _, backup := range backups { + if backup.GetGlobalId() == backupId { + return backup, nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (rds *SDBInstance) GetIDBInstanceBackups() ([]cloudprovider.ICloudDBInstanceBackup, error) { + backups, err := rds.region.GetDBInstanceBackups(rds.Id, "") + if err != nil { + return nil, err + } + + ibackups := []cloudprovider.ICloudDBInstanceBackup{} + for i := 0; i < len(backups); i++ { + backups[i].region = rds.region + ibackups = append(ibackups, &backups[i]) + } + return ibackups, nil +} + +func (backup *SDBInstanceBackup) Refresh() error { + backups, err := backup.region.GetDBInstanceBackups(backup.InstanceId, backup.Id) + if err != nil { + return err + } + if len(backups) == 0 { + return cloudprovider.ErrNotFound + } + return jsonutils.Update(backup, backups[0]) +} + +func (rds *SDBInstance) CreateIBackup(conf *cloudprovider.SDBInstanceBackupCreateConfig) (string, error) { + backupId, err := rds.region.CreateDBInstanceBackup(rds.Id, conf.Name, conf.Description, conf.Databases) + if err != nil { + return "", err + } + backup, err := rds.region.GetIDBInstanceBackupById(backupId) + if err != nil { + return "", errors.Wrap(err, "region.GetIDBInstanceBackupById") + } + cloudprovider.WaitStatus(backup, api.DBINSTANCE_BACKUP_READY, time.Second*3, time.Minute*30) + return backupId, nil +} + +func (region *SRegion) CreateDBInstanceBackup(instanceId string, name string, descrition string, databases []string) (string, error) { + params := map[string]interface{}{ + "instance_id": instanceId, + "name": name, + "description": descrition, + } + if len(databases) > 0 { + dbs := []map[string]string{} + for _, database := range databases { + dbs = append(dbs, map[string]string{"name": database}) + } + params["databases"] = dbs + } + backup := SDBInstanceBackup{} + err := region.rdsCreate("backups", params, backup) + if err != nil { + return "", errors.Wrap(err, "DBInstanceBackup.Create") + } + backupId := backup.Id + if err != nil { + return "", errors.Wrap(err, "resp.GetBackupId") + } + return backupId, nil +} + +func (self *SDBInstanceBackup) CreateICloudDBInstance(opts *cloudprovider.SManagedDBInstanceCreateConfig) (cloudprovider.ICloudDBInstance, error) { + opts.BackupId = self.Id + return self.region.CreateIDBInstance(opts) +} diff --git a/pkg/multicloud/hcs/dbinstance_database.go b/pkg/multicloud/hcs/dbinstance_database.go new file mode 100644 index 0000000000..5c06d6317f --- /dev/null +++ b/pkg/multicloud/hcs/dbinstance_database.go @@ -0,0 +1,72 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SDBInstanceDatabase struct { + instance *SDBInstance + multicloud.SDBInstanceDatabaseBase + multicloud.HuaweiTags + + Name string + CharacterSet string +} + +func (database *SDBInstanceDatabase) GetId() string { + return database.Name +} + +func (database *SDBInstanceDatabase) GetGlobalId() string { + return database.Name +} + +func (database *SDBInstanceDatabase) GetName() string { + return database.Name +} + +func (database *SDBInstanceDatabase) GetStatus() string { + return api.DBINSTANCE_DATABASE_RUNNING +} + +func (database *SDBInstanceDatabase) GetCharacterSet() string { + return database.CharacterSet +} + +func (database *SDBInstanceDatabase) Delete() error { + return database.instance.region.DeleteDBInstanceDatabase(database.instance.Id, database.Name) +} + +func (region *SRegion) DeleteDBInstanceDatabase(instanceId, database string) error { + resource := fmt.Sprintf("instances/%s/database/%s", instanceId, database) + err := region.rdsDelete(resource) + return err +} + +// rds查询数据库列表 +func (region *SRegion) GetDBInstanceDatabases(instanceId string) ([]SDBInstanceDatabase, error) { + resource := fmt.Sprintf("%s/database/detail", instanceId) + databases := []SDBInstanceDatabase{} + err := region.rdsList(resource, nil, databases) + if err != nil { + return nil, err + } + return databases, nil +} diff --git a/pkg/multicloud/hcs/dbinstance_parameter.go b/pkg/multicloud/hcs/dbinstance_parameter.go new file mode 100644 index 0000000000..831ca7a330 --- /dev/null +++ b/pkg/multicloud/hcs/dbinstance_parameter.go @@ -0,0 +1,52 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +type SDBInstanceParameter struct { + instance *SDBInstance + + Name string + Value string + RestartRequired bool + Readonly bool + ValueRange string + Type string + Description string +} + +func (region *SRegion) GetDBInstanceParameters(dbinstanceId string) ([]SDBInstanceParameter, error) { + paramters := []SDBInstanceParameter{} + err := region.rdsList("configurations", nil, paramters) + if err != nil { + return nil, err + } + return paramters, nil +} + +func (param *SDBInstanceParameter) GetGlobalId() string { + return param.Name +} + +func (param *SDBInstanceParameter) GetKey() string { + return param.Name +} + +func (param *SDBInstanceParameter) GetValue() string { + return param.Value +} + +func (param *SDBInstanceParameter) GetDescription() string { + return param.Description +} diff --git a/pkg/multicloud/hcs/dbinstance_privilege.go b/pkg/multicloud/hcs/dbinstance_privilege.go new file mode 100644 index 0000000000..19ba785528 --- /dev/null +++ b/pkg/multicloud/hcs/dbinstance_privilege.go @@ -0,0 +1,43 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + api "yunion.io/x/onecloud/pkg/apis/compute" +) + +type SDatabasePrivilege struct { + account *SDBInstanceAccount + + Name string + Readonly bool +} + +func (privilege *SDatabasePrivilege) GetGlobalId() string { + return fmt.Sprintf("%s/%s", privilege.account.Name, privilege.Name) +} + +func (privilege *SDatabasePrivilege) GetPrivilege() string { + if privilege.Readonly { + return api.DATABASE_PRIVILEGE_R + } + return api.DATABASE_PRIVILEGE_RW +} + +func (privilege *SDatabasePrivilege) GetDBName() string { + return privilege.Name +} diff --git a/pkg/multicloud/hcs/dbinstance_skus.go b/pkg/multicloud/hcs/dbinstance_skus.go new file mode 100644 index 0000000000..ffa779f24d --- /dev/null +++ b/pkg/multicloud/hcs/dbinstance_skus.go @@ -0,0 +1,273 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + "strings" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SDBInstanceSku struct { + Vcpus int + Ram int //单位GB + SpecCode string + InstanceMode string //实例模型 + + StorageType string + Az1 string + Az2 string + Engine string + EngineVersion string +} + +func (self *SDBInstanceSku) GetName() string { + return self.SpecCode +} + +func (self *SDBInstanceSku) GetGlobalId() string { + return fmt.Sprintf("%s-%s-%s-%s-%s-%s", self.Engine, self.EngineVersion, self.InstanceMode, self.SpecCode, self.StorageType, self.GetZoneId()) +} + +func (self *SDBInstanceSku) GetCategory() string { + return self.InstanceMode +} + +func (self *SDBInstanceSku) GetDiskSizeStep() int { + return 10 +} + +func (self *SDBInstanceSku) GetIOPS() int { + return 0 +} + +func (self *SDBInstanceSku) GetTPS() int { + return 0 +} + +func (self *SDBInstanceSku) GetZone1Id() string { + return self.Az1 +} + +func (self *SDBInstanceSku) GetZone2Id() string { + return self.Az2 +} + +func (self *SDBInstanceSku) GetZone3Id() string { + return "" +} + +func (self *SDBInstanceSku) GetZoneId() string { + zones := []string{} + for _, zone := range []string{self.Az1, self.Az2} { + if len(zone) > 0 { + zones = append(zones, zone) + } + } + return strings.Join(zones, ",") +} + +func (self *SDBInstanceSku) GetMaxConnections() int { + return 0 +} + +func (self *SDBInstanceSku) GetMaxDiskSizeGb() int { + return 4000 +} + +func (self *SDBInstanceSku) GetMinDiskSizeGb() int { + return 40 +} + +func (self *SDBInstanceSku) GetVcpuCount() int { + return self.Vcpus +} + +func (self *SDBInstanceSku) GetVmemSizeMb() int { + return self.Ram * 1024 +} + +func (self *SDBInstanceSku) GetStorageType() string { + return self.StorageType +} + +func (sku *SDBInstanceSku) GetEngine() string { + return sku.Engine +} + +func (sku *SDBInstanceSku) GetEngineVersion() string { + return sku.EngineVersion +} + +func (self *SDBInstanceSku) GetStatus() string { + return api.DBINSTANCE_SKU_AVAILABLE +} + +func (sku *SDBInstanceSku) GetQPS() int { + return 0 +} + +type SDBInstanceFlavor struct { + Vcpus int + Ram int //单位GB + SpecCode string + InstanceMode string //实例模型 + + StorageType string + AzStatus map[string]string + Engine string + EngineVersion string +} + +func (self *SDBInstanceFlavor) GetISkus() []cloudprovider.ICloudDBInstanceSku { + ret := []cloudprovider.ICloudDBInstanceSku{} + switch self.InstanceMode { + case api.HUAWEI_DBINSTANCE_CATEGORY_HA: + for az1, status1 := range self.AzStatus { + if status1 != "normal" { + continue + } + for az2, status2 := range self.AzStatus { + if status2 != "normal" { + continue + } + sku := &SDBInstanceSku{ + Vcpus: self.Vcpus, + Ram: self.Ram, + SpecCode: self.SpecCode, + InstanceMode: self.InstanceMode, + StorageType: self.StorageType, + Az1: az1, + Az2: az2, + Engine: self.Engine, + EngineVersion: self.EngineVersion, + } + ret = append(ret, sku) + } + } + case api.HUAWEI_DBINSTANCE_CATEGORY_SINGLE: + for az, status := range self.AzStatus { + if status != "normal" { + continue + } + sku := &SDBInstanceSku{ + Vcpus: self.Vcpus, + Ram: self.Ram, + SpecCode: self.SpecCode, + InstanceMode: self.InstanceMode, + StorageType: self.StorageType, + Az1: az, + Engine: self.Engine, + EngineVersion: self.EngineVersion, + } + ret = append(ret, sku) + } + } + return ret +} + +func (self *SRegion) GetIDBInstanceSkus() ([]cloudprovider.ICloudDBInstanceSku, error) { + skus, err := self.GetDBInstanceSkus() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudDBInstanceSku{} + for i := range skus { + ret = append(ret, skus[i].GetISkus()...) + } + return ret, nil +} + +type SDBInstanceDatastore struct { + Id string + Name string +} + +func (self *SRegion) GetDBInstanceSkus() ([]SDBInstanceFlavor, error) { + skus := []SDBInstanceFlavor{} + for _, engine := range []string{api.DBINSTANCE_TYPE_MYSQL, api.DBINSTANCE_TYPE_POSTGRESQL, api.DBINSTANCE_TYPE_SQLSERVER} { + stores, err := self.GetDBInstanceDatastores(engine) + if err != nil { + return nil, err + } + for _, version := range stores { + storages, err := self.GetDBInstanceStorages(engine, version.Name) + if err != nil { + return nil, errors.Wrapf(err, "GetDBInstanceStorages(%s,%s)", engine, version.Name) + } + flavors, err := self.GetDBInstanceFlavors(engine, version.Name) + if err != nil { + return nil, errors.Wrapf(err, "GetDBInstanceFlavors(%s, %s)", engine, version.Name) + } + for i := range flavors { + flavors[i].Engine = engine + flavors[i].EngineVersion = version.Name + for j := range storages { + flavors[i].StorageType = storages[j].Name + flavors[i].AzStatus = storages[j].AzStatus + skus = append(skus, flavors[i]) + } + } + } + } + return skus, nil +} + +// rds规格信息 +func (region *SRegion) GetDBInstanceFlavors(engine string, version string) ([]SDBInstanceFlavor, error) { + flavors := []SDBInstanceFlavor{} + query := url.Values{} + query.Add("version_name", version) + resource := fmt.Sprintf("%s/%s", "flavor", engine) + err := region.rdsList(resource, query, flavors) + if err != nil { + return nil, err + } + return flavors, nil +} + +// rds数据库引擎版本信息 +func (region *SRegion) GetDBInstanceDatastores(engine string) ([]SDBInstanceDatastore, error) { + stores := []SDBInstanceDatastore{} + resource := fmt.Sprintf("%s/%s", "datastores", engine) + err := region.rdsList(resource, nil, stores) + if err != nil { + return nil, err + } + return stores, nil +} + +type SDBInstanceStorage struct { + Name string + AzStatus map[string]string +} + +// rds数据库磁盘类型 +func (region *SRegion) GetDBInstanceStorages(engine, engineVersion string) ([]SDBInstanceStorage, error) { + storages := []SDBInstanceStorage{} + resource := fmt.Sprintf("%s/%s", "storage-type", engine) + query := url.Values{} + query.Add("version_name", engineVersion) + err := region.rdsList(resource, query, storages) + if err != nil { + return nil, err + } + return storages, nil +} diff --git a/pkg/multicloud/hcs/disk.go b/pkg/multicloud/hcs/disk.go new file mode 100644 index 0000000000..784cf8d0ed --- /dev/null +++ b/pkg/multicloud/hcs/disk.go @@ -0,0 +1,438 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + + billing_api "yunion.io/x/onecloud/pkg/apis/billing" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +/* +华为云云硬盘 +======创建========== +1.磁盘只能挂载到同一可用区的云服务器内，创建后不支持更换可用区 +2.计费模式 包年包月/按需计费 +3.*支持自动备份 + + +共享盘 和 普通盘：https://support.huaweicloud.com/productdesc-evs/zh-cn_topic_0032860759.html +根据是否支持挂载至多台云服务器可以将云硬盘分为非共享云硬盘和共享云硬盘。 +一个非共享云硬盘只能挂载至一台云服务器，而一个共享云硬盘可以同时挂载至多台云服务器。 +单个共享云硬盘最多可同时挂载给16个云服务器。目前，共享云硬盘只适用于数据盘，不支持系统盘。 +*/ + +type Attachment struct { + ServerId string `json:"server_id"` + AttachmentId string `json:"attachment_id"` + AttachedAt string `json:"attached_at"` + HostName string `json:"host_name"` + VolumeId string `json:"volume_id"` + Device string `json:"device"` + Id string `json:"id"` +} + +type DiskMeta struct { + ResourceSpecCode string `json:"resourceSpecCode"` + Billing string `json:"billing"` + ResourceType string `json:"resourceType"` + AttachedMode string `json:"attached_mode"` + Readonly string `json:"readonly"` +} + +type VolumeImageMetadata struct { + QuickStart string `json:"__quick_start"` + ContainerFormat string `json:"container_format"` + MinRAM string `json:"min_ram"` + ImageName string `json:"image_name"` + ImageId string `json:"image_id"` + OSType string `json:"__os_type"` + OSFeatureList string `json:"__os_feature_list"` + MinDisk string `json:"min_disk"` + SupportKVM string `json:"__support_kvm"` + VirtualEnvType string `json:"virtual_env_type"` + SizeGB string `json:"size"` + OSVersion string `json:"__os_version"` + OSBit string `json:"__os_bit"` + SupportKVMHi1822Hiovs string `json:"__support_kvm_hi1822_hiovs"` + SupportXen string `json:"__support_xen"` + Description string `json:"__description"` + Imagetype string `json:"__imagetype"` + DiskFormat string `json:"disk_format"` + ImageSourceType string `json:"__image_source_type"` + Checksum string `json:"checksum"` + Isregistered string `json:"__isregistered"` + HwVifMultiqueueEnabled string `json:"hw_vif_multiqueue_enabled"` + Platform string `json:"__platform"` +} + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0124881427.html +type SDisk struct { + multicloud.SDisk + multicloud.HuaweiDiskTags + //details *SResourceDetail + + region *SRegion + + Id string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Attachments []Attachment `json:"attachments"` + Description string `json:"description"` + SizeGB int `json:"size"` + Metadata DiskMeta `json:"metadata"` + Encrypted bool `json:"encrypted"` + Bootable string `json:"bootable"` + Multiattach bool `json:"multiattach"` + AvailabilityZone string `json:"availability_zone"` + SourceVolid string `json:"source_volid"` + SnapshotId string `json:"snapshot_id"` + CreatedAt time.Time `json:"created_at"` + VolumeType string `json:"volume_type"` + VolumeImageMetadata VolumeImageMetadata `json:"volume_image_metadata"` + ReplicationStatus string `json:"replication_status"` + UserId string `json:"user_id"` + ConsistencygroupId string `json:"consistencygroup_id"` + UpdatedAt string `json:"updated_at"` + EnterpriseProjectId string + + ExpiredTime time.Time +} + +func (self *SDisk) GetId() string { + return self.Id +} + +func (self *SDisk) GetName() string { + if len(self.Name) == 0 { + return self.Id + } + + return self.Name +} + +func (self *SDisk) GetGlobalId() string { + return self.Id +} + +func (self *SDisk) GetStatus() string { + // https://support.huaweicloud.com/api-evs/zh-cn_topic_0051803385.html + switch self.Status { + case "creating", "downloading": + return api.DISK_ALLOCATING + case "available", "in-use": + return api.DISK_READY + case "error": + return api.DISK_ALLOC_FAILED + case "attaching": + return api.DISK_ATTACHING + case "detaching": + return api.DISK_DETACHING + case "restoring-backup": + return api.DISK_REBUILD + case "backing-up": + return api.DISK_BACKUP_STARTALLOC // ? + case "error_restoring": + return api.DISK_BACKUP_ALLOC_FAILED + case "uploading": + return api.DISK_SAVING //? + case "extending": + return api.DISK_RESIZING + case "error_extending": + return api.DISK_ALLOC_FAILED // ? + case "deleting": + return api.DISK_DEALLOC //? + case "error_deleting": + return api.DISK_DEALLOC_FAILED // ? + case "rollbacking": + return api.DISK_REBUILD + case "error_rollbacking": + return api.DISK_UNKNOWN + default: + return api.DISK_UNKNOWN + } +} + +func (self *SDisk) Refresh() error { + ret, err := self.region.GetDisk(self.Id) + if err != nil { + return err + } + return jsonutils.Update(self, ret) +} + +func (self *SDisk) GetBillingType() string { + return billing_api.BILLING_TYPE_POSTPAID +} + +func (self *SDisk) GetCreatedAt() time.Time { + return self.CreatedAt +} + +func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return nil, cloudprovider.ErrNotImplemented + // return self.storage, nil +} + +func (self *SDisk) GetDiskFormat() string { + return "vhd" +} + +func (self *SDisk) GetDiskSizeMB() int { + return int(self.SizeGB * 1024) +} + +func (self *SDisk) GetIsAutoDelete() bool { + return false +} + +func (self *SDisk) GetTemplateId() string { + return self.VolumeImageMetadata.ImageId +} + +// Bootable 表示硬盘是否为启动盘。 +// 启动盘 != 系统盘(必须是启动盘且挂载在root device上) +func (self *SDisk) GetDiskType() string { + if self.Bootable == "true" { + return api.DISK_TYPE_SYS + } + return api.DISK_TYPE_DATA +} + +func (self *SDisk) GetFsFormat() string { + return "" +} + +func (self *SDisk) GetIsNonPersistent() bool { + return false +} + +func (self *SDisk) GetDriver() string { + return "scsi" +} + +func (self *SDisk) GetCacheMode() string { + return "none" +} + +func (self *SDisk) GetMountpoint() string { + if len(self.Attachments) > 0 { + return self.Attachments[0].Device + } + return "" +} + +func (self *SDisk) GetMountServerId() string { + if len(self.Attachments) > 0 { + return self.Attachments[0].ServerId + } + return "" +} + +func (self *SDisk) GetAccessPath() string { + return "" +} + +func (self *SDisk) Delete(ctx context.Context) error { + return self.region.DeleteDisk(self.Id) +} + +func (self *SDisk) CreateISnapshot(ctx context.Context, name string, desc string) (cloudprovider.ICloudSnapshot, error) { + snap, err := self.region.CreateSnapshot(self.Id, name, desc) + if err != nil { + return nil, err + } + return snap, nil +} + +func (self *SDisk) GetISnapshot(id string) (cloudprovider.ICloudSnapshot, error) { + snapshot, err := self.region.GetSnapshot(id) + if err != nil { + return nil, err + } + return snapshot, nil +} + +func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { + snapshots, err := self.region.GetSnapshots(self.Id, "") + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudSnapshot{} + for i := 0; i < len(snapshots); i++ { + snapshots[i].region = self.region + ret = append(ret, &snapshots[i]) + } + return ret, nil +} + +func (self *SDisk) Resize(ctx context.Context, sizeMB int64) error { + return self.region.ResizeDisk(self.GetId(), sizeMB/1024) +} + +func (self *SDisk) Detach() error { + return self.region.DetachDisk(self.GetMountServerId(), self.GetId()) +} + +func (self *SDisk) Attach(device string) error { + return self.region.AttachDisk(self.GetMountServerId(), self.GetId(), device) +} + +// 在线卸载磁盘 https://support.huaweicloud.com/usermanual-ecs/zh-cn_topic_0036046828.html +// 对于挂载在系统盘盘位（也就是“/dev/sda”或“/dev/vda”挂载点）上的磁盘，当前仅支持离线卸载 +func (self *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) { + mountpoint := self.GetMountpoint() + if len(mountpoint) > 0 { + err := self.Detach() + if err != nil { + return "", err + } + } + + err := self.region.ResetDisk(self.Id, snapshotId) + if err != nil { + return self.Id, err + } + err = cloudprovider.WaitStatus(self, api.DISK_READY, 5*time.Second, 300*time.Second) + if err != nil { + return "", err + } + if len(mountpoint) > 0 { + err := self.Attach(mountpoint) + if err != nil { + return "", err + } + } + return self.Id, nil +} + +// 华为云不支持重置 +func (self *SDisk) Rebuild(ctx context.Context) error { + return cloudprovider.ErrNotSupported +} + +func (self *SRegion) GetDisk(id string) (*SDisk, error) { + ret := &SDisk{region: self} + res := fmt.Sprintf("volumes/%s", id) + return ret, self.evsGet(res, ret) +} + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0058762430.html +func (self *SRegion) GetDisks(zoneId string) ([]SDisk, error) { + params := url.Values{} + if len(zoneId) > 0 { + params.Set("availability_zone", zoneId) + } + disks := []SDisk{} + return disks, self.evsList("volumes/detail", params, &disks) +} + +func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, snapshotId string, desc string, projectId string) (*SDisk, error) { + volume := map[string]interface{}{ + "name": name, + "availability_zone": zoneId, + "description": desc, + "volume_type": category, + "size": sizeGb, + } + if len(snapshotId) > 0 { + volume["snapshot_id"] = snapshotId + } + if len(projectId) > 0 { + volume["enterprise_project_id"] = projectId + } + params := map[string]interface{}{ + "volume": volume, + } + ret := &SDisk{region: self} + return ret, self.evsCreate("volumes", params, ret) +} + +func (self *SRegion) DeleteDisk(id string) error { + res := fmt.Sprintf("volumes/%s", id) + return self.evsDelete(res) +} + +func (self *SRegion) ResizeDisk(id string, sizeGb int64) error { + params := map[string]interface{}{ + "os_extend": map[string]interface{}{ + "new_size": sizeGb, + }, + } + res := fmt.Sprintf("volumes/%s", id) + return self.evsPerform(res, "action", params) +} + +func (self *SRegion) ResetDisk(diskId, snapshotId string) error { + params := map[string]interface{}{ + "volume_id": diskId, + } + res := fmt.Sprintf("os-vendor-snapshots/%s", snapshotId) + return self.evsPerform(res, "rollback", params) +} + +func (self *SDisk) GetProjectId() string { + return self.EnterpriseProjectId +} + +func (self *SRegion) DetachDisk(instanceId string, diskId string) error { + res := fmt.Sprintf("servers/%s/os-volume_attachments/%s", instanceId, diskId) + return self.delete("ecs", "v2", res) +} + +func (self *SRegion) AttachDisk(instanceId string, diskId string, device string) error { + params := map[string]interface{}{ + "volumeAttachment": map[string]interface{}{ + "volumeId": instanceId, + "device": device, + }, + } + res := fmt.Sprintf("servers/%s/os-volume_attachments", instanceId) + return self.perform("ecs", "v2", res, "action", params, nil) +} + +type SDiskType struct { + ExtraSpecs ExtraSpecs `json:"extra_specs"` + Name string `json:"name"` + QosSpecsId string `json:"qos_specs_id"` + Id string `json:"id"` + IsPublic bool `json:"is_public"` +} + +func (self *SDiskType) IsAvaliableInZone(zoneId string) bool { + if strings.Contains(self.ExtraSpecs.HwAvailabilityZone, zoneId) { + return true + } + return false +} + +type ExtraSpecs struct { + VolumeBackendName string `json:"volume_backend_name"` + HwAvailabilityZone string `json:"HW:availability-zone"` +} + +func (self *SRegion) GetDiskTypes() ([]SDiskType, error) { + ret := []SDiskType{} + return ret, self.get("evs", "v3", "types", &ret) +} diff --git a/pkg/multicloud/hcs/doc.go b/pkg/multicloud/hcs/doc.go new file mode 100644 index 0000000000..d68c43049b --- /dev/null +++ b/pkg/multicloud/hcs/doc.go @@ -0,0 +1 @@ +package hcs // import "yunion.io/x/onecloud/pkg/multicloud/hcs" diff --git a/pkg/multicloud/hcs/eip.go b/pkg/multicloud/hcs/eip.go new file mode 100644 index 0000000000..2200eb6350 --- /dev/null +++ b/pkg/multicloud/hcs/eip.go @@ -0,0 +1,434 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + billing_api "yunion.io/x/onecloud/pkg/apis/billing" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type TInternetChargeType string + +const ( + InternetChargeByTraffic = TInternetChargeType("traffic") + InternetChargeByBandwidth = TInternetChargeType("bandwidth") +) + +type Bandwidth struct { + Id string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + ShareType string `json:"share_type"` + PublicipInfo []PublicipInfo `json:"publicip_info"` + TenantId string `json:"tenant_id"` + BandwidthType string `json:"bandwidth_type"` + ChargeMode string `json:"charge_mode"` + BillingInfo string `json:"billing_info"` + EnterpriseProjectId string `json:"enterprise_project_id"` +} + +type PublicipInfo struct { + PublicipId string `json:"publicip_id"` + PublicipAddress string `json:"publicip_address"` + PublicipType string `json:"publicip_type"` + IPVersion int64 `json:"ip_version"` +} + +type SProfile struct { + UserId string `json:"user_id"` + ProductId string `json:"product_id"` + RegionId string `json:"region_id"` + OrderId string `json:"order_id"` +} + +type SEip struct { + multicloud.SEipBase + multicloud.HcsTags + + region *SRegion + port *Port + + Alias string + Id string `json:"id"` + Status string `json:"status"` + Profile *SProfile `json:"profile,omitempty"` + Type string `json:"type"` + PublicIPAddress string `json:"public_ip_address"` + PrivateIPAddress string `json:"private_ip_address"` + TenantId string `json:"tenant_id"` + CreateTime time.Time `json:"create_time"` + BandwidthId string `json:"bandwidth_id"` + BandwidthShareType string `json:"bandwidth_share_type"` + BandwidthSize int64 `json:"bandwidth_size"` + BandwidthName string `json:"bandwidth_name"` + EnterpriseProjectId string `json:"enterprise_project_id"` + IPVersion int64 `json:"ip_version"` + PortId string `json:"port_id"` +} + +func (self *SEip) GetId() string { + return self.Id +} + +func (self *SEip) GetName() string { + if len(self.Alias) > 0 { + return self.Alias + } + return self.PublicIPAddress +} + +func (self *SEip) GetGlobalId() string { + return self.Id +} + +func (self *SEip) GetStatus() string { + // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090598.html + switch self.Status { + case "ACTIVE", "DOWN", "ELB": + return api.EIP_STATUS_READY + case "PENDING_CREATE", "NOTIFYING": + return api.EIP_STATUS_ALLOCATE + case "BINDING": + return api.EIP_STATUS_ALLOCATE + case "BIND_ERROR": + return api.EIP_STATUS_ALLOCATE_FAIL + case "PENDING_DELETE", "NOTIFY_DELETE": + return api.EIP_STATUS_DEALLOCATE + default: + return api.EIP_STATUS_UNKNOWN + } +} + +func (self *SEip) Refresh() error { + if self.IsEmulated() { + return nil + } + new, err := self.region.GetEip(self.Id) + if err != nil { + return err + } + return jsonutils.Update(self, new) +} + +func (self *SEip) IsEmulated() bool { + return false +} + +func (self *SEip) GetIpAddr() string { + return self.PublicIPAddress +} + +func (self *SEip) GetMode() string { + return api.EIP_MODE_STANDALONE_EIP +} + +func (self *SEip) GetPort() *Port { + if len(self.PortId) == 0 { + return nil + } + + if self.port != nil { + return self.port + } + + port, err := self.region.GetPort(self.PortId) + if err != nil { + return nil + } + self.port = port + return self.port +} + +func (self *SEip) GetAssociationType() string { + if len(self.PortId) == 0 { + return "" + } + port, err := self.region.GetPort(self.PortId) + if err != nil { + log.Errorf("Get eip %s port %s error: %v", self.Id, self.PortId, err) + return "" + } + + if strings.HasPrefix(port.DeviceOwner, "compute") { + return api.EIP_ASSOCIATE_TYPE_SERVER + } + + switch port.DeviceOwner { + case "neutron:LOADBALANCER", "neutron:LOADBALANCERV2": + return api.EIP_ASSOCIATE_TYPE_LOADBALANCER + case "network:nat_gateway": + return api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY + default: + return port.DeviceOwner + } +} + +func (self *SEip) GetAssociationExternalId() string { + // network/0273a359d61847fc83405926c958c746/ext-floatingips?tenantId=0273a359d61847fc83405926c958c746&limit=2000 + // 只能通过 port id 反查device id. + if len(self.PortId) > 0 { + port, _ := self.region.GetPort(self.PortId) + return port.DeviceId + } + return "" +} + +func (self *SEip) GetBandwidth() int { + return int(self.BandwidthSize) // Mb +} + +func (self *SEip) GetINetworkId() string { + return "" +} + +func (self *SEip) GetInternetChargeType() string { + // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090603.html + bandwidth, err := self.region.GetEipBandwidth(self.BandwidthId) + if err != nil { + return api.EIP_CHARGE_TYPE_BY_TRAFFIC + } + if bandwidth.ChargeMode == "traffic" { + return api.EIP_CHARGE_TYPE_BY_TRAFFIC + } + return api.EIP_CHARGE_TYPE_BY_BANDWIDTH +} + +func (self *SEip) GetBillingType() string { + return billing_api.BILLING_TYPE_POSTPAID +} + +func (self *SEip) GetCreatedAt() time.Time { + return self.CreateTime +} + +func (self *SEip) GetExpiredAt() time.Time { + return time.Time{} +} + +func (self *SEip) Delete() error { + return self.region.DeallocateEIP(self.Id) +} + +func (self *SEip) Associate(conf *cloudprovider.AssociateConfig) error { + portId, err := self.region.GetInstancePortId(conf.InstanceId) + if err != nil { + return err + } + + if len(self.PortId) > 0 { + if self.PortId == portId { + return nil + } + + return fmt.Errorf("eip %s aready associate with port %s", self.GetId(), self.PortId) + } + + err = self.region.AssociateEipWithPortId(self.Id, portId) + if err != nil { + return err + } + + err = cloudprovider.WaitStatusWithDelay(self, api.EIP_STATUS_READY, 10*time.Second, 10*time.Second, 180*time.Second) + return err +} + +func (self *SEip) Dissociate() error { + err := self.region.DissociateEip(self.Id) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, api.EIP_STATUS_READY, 10*time.Second, 180*time.Second) +} + +func (self *SEip) ChangeBandwidth(bw int) error { + return self.region.UpdateEipBandwidth(self.BandwidthId, bw) +} + +func (self *SRegion) GetInstancePortId(instanceId string) (string, error) { + // 目前只绑定一个网卡 + // todo: 还需要按照ports状态进行过滤 + ports, err := self.GetPorts(instanceId) + if err != nil { + return "", err + } + + if len(ports) == 0 { + return "", fmt.Errorf("AssociateEip instance %s port is empty", instanceId) + } + + return ports[0].Id, nil +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090596.html +func (self *SRegion) AllocateEIP(name string, bwMbps int, chargeType TInternetChargeType, bgpType string, projectId string) (*SEip, error) { + params := map[string]interface{}{ + "bandwidth": map[string]interface{}{ + "name": name, + "size": bwMbps, + "share_type": "PER", + "charge_mode": chargeType, + }, + "publicip": map[string]interface{}{ + "type": bgpType, + "ip_version": 4, + "alias": name, + }, + } + if len(projectId) > 0 { + params["enterprise_project_id"] = projectId + } + eip := &SEip{region: self} + return eip, self.vpcCreate("publicips", params, eip) +} + +func (self *SRegion) GetEip(eipId string) (*SEip, error) { + eip := &SEip{region: self} + return eip, self.vpcGet("publicips/"+eipId, eip) +} + +func (self *SRegion) DeallocateEIP(eipId string) error { + return self.vpcDelete("publicips/" + eipId) +} + +func (self *SRegion) AssociateEip(eipId string, instanceId string) error { + portId, err := self.GetInstancePortId(instanceId) + if err != nil { + return err + } + return self.AssociateEipWithPortId(eipId, portId) +} + +func (self *SRegion) AssociateEipWithPortId(eipId string, portId string) error { + params := map[string]interface{}{ + "publicip": map[string]interface{}{ + "port_id": portId, + }, + } + return self.vpcUpdate("publicips/"+eipId, params) +} + +func (self *SRegion) DissociateEip(eipId string) error { + return self.AssociateEipWithPortId(eipId, "") +} + +func (self *SRegion) UpdateEipBandwidth(bandwidthId string, bw int) error { + params := map[string]interface{}{ + "bandwidth": map[string]interface{}{ + "size": bw, + }, + } + return self.vpcUpdate("bandwidths/"+bandwidthId, params) +} + +func (self *SRegion) GetEipBandwidth(id string) (*Bandwidth, error) { + ret := &Bandwidth{} + return ret, self.vpcGet("bandwidths/"+id, ret) +} + +func (self *SEip) GetProjectId() string { + return self.EnterpriseProjectId +} + +func (self *SRegion) GetEips(portId string, addrs []string) ([]SEip, error) { + query := url.Values{} + for _, addr := range addrs { + query.Add("public_ip_address", addr) + } + if len(portId) > 0 { + query.Set("port_id", portId) + } + eips := []SEip{} + return eips, self.vpcList("publicips", query, &eips) +} + +type SEipType struct { + Id string + Type string + Name string + Group string +} + +func (self *SRegion) GetEipTypes() ([]SEipType, error) { + query := url.Values{} + ret := []SEipType{} + return ret, self.vpcList("publicip_types", query, &ret) +} + +func (self *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) { + eips, err := self.GetEips("", nil) + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudEIP{} + for i := 0; i < len(eips); i += 1 { + eips[i].region = self + ret = append(ret, &eips[i]) + } + return ret, nil +} + +func (self *SRegion) GetIEipById(id string) (cloudprovider.ICloudEIP, error) { + eip, err := self.GetEip(id) + if err != nil { + return nil, err + } + return eip, nil +} + +func (self *SRegion) CreateEIP(eip *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) { + var ctype TInternetChargeType + switch eip.ChargeType { + case api.EIP_CHARGE_TYPE_BY_TRAFFIC: + ctype = InternetChargeByTraffic + case api.EIP_CHARGE_TYPE_BY_BANDWIDTH: + ctype = InternetChargeByBandwidth + } + + // todo: 如何避免hardcode。集成到cloudmeta服务中？ + if len(eip.BGPType) == 0 { + types, err := self.GetEipTypes() + if err != nil { + return nil, errors.Wrapf(err, "GetEipTypes") + } + if len(types) > 0 { + eip.BGPType = types[0].Type + } + } + + // 华为云EIP名字最大长度64 + if len(eip.Name) > 64 { + eip.Name = eip.Name[:64] + } + + ieip, err := self.AllocateEIP(eip.Name, eip.BandwidthMbps, ctype, eip.BGPType, eip.ProjectId) + ieip.region = self + if err != nil { + return nil, err + } + + err = cloudprovider.WaitStatus(ieip, api.EIP_STATUS_READY, 5*time.Second, 60*time.Second) + return ieip, err +} diff --git a/pkg/multicloud/hcs/hcs.go b/pkg/multicloud/hcs/hcs.go new file mode 100644 index 0000000000..5c167f5f52 --- /dev/null +++ b/pkg/multicloud/hcs/hcs.go @@ -0,0 +1,771 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcso/client/auth" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +const ( + VERSION_SPEC = "x-version" + + CLOUD_PROVIDER_HCS = api.CLOUD_PROVIDER_HCS + CLOUD_PROVIDER_HCS_CN = "华为云Stack" + CLOUD_PROVIDER_HCS_EN = "HCS" + + HCS_API_VERSION = "" +) + +type HcsConfig struct { + cpcfg cloudprovider.ProviderConfig + authUrl string + + projectId string // 华为云项目ID. + accessKey string + accessSecret string + + debug bool +} + +func NewHcsConfig(accessKey, accessSecret, projectId, url string) *HcsConfig { + cfg := &HcsConfig{ + projectId: projectId, + accessKey: accessKey, + accessSecret: accessSecret, + authUrl: url, + } + + return cfg +} + +func (cfg *HcsConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *HcsConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *HcsConfig) Debug(debug bool) *HcsConfig { + cfg.debug = debug + return cfg +} + +type SHcsClient struct { + *HcsConfig + + signer auth.Signer + + isMainProject bool // whether the project is the main project in the region + + domainId string + projectName string + + regions []SRegion + projects []SProject + buckets []SBucket + + defaultRegion string + + httpClient *http.Client + lock sThrottlingThreshold +} + +func NewHcsClient(cfg *HcsConfig) (*SHcsClient, error) { + client := SHcsClient{ + HcsConfig: cfg, + lock: sThrottlingThreshold{locked: false, lockTime: time.Time{}}, + } + if len(client.projectId) > 0 { + project, err := client.GetProjectById(client.projectId) + if err != nil { + return nil, err + } + client.domainId = project.DomainId + client.projectName = project.Name + } + return &client, client.fetchRegions() +} + +func (self *SHcsClient) GetAccountId() string { + return self.authUrl +} + +func (self *SHcsClient) GetRegion(id string) (*SRegion, error) { + for i := range self.regions { + if self.regions[i].GetGlobalId() == id || self.regions[i].GetId() == id { + self.regions[i].client = self + return &self.regions[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SHcsClient) GetCloudRegionExternalIdPrefix() string { + if len(self.projectId) > 0 { + if iregions := self.GetIRegions(); len(iregions) > 0 { + return iregions[0].GetGlobalId() + } + } + return CLOUD_PROVIDER_HCS +} + +type akClient struct { + client *http.Client + signer *Signer +} + +func (self *SHcsClient) getDefaultClient() *http.Client { + if self.httpClient != nil { + return self.httpClient + } + self.httpClient = self.cpcfg.AdaptiveTimeoutHttpClient() + ts, _ := self.httpClient.Transport.(*http.Transport) + self.httpClient.Transport = cloudprovider.GetCheckTransport(ts, func(req *http.Request) (func(resp *http.Response), error) { + service, method, path := strings.Split(req.URL.Host, ".")[0], req.Method, req.URL.Path + respCheck := func(resp *http.Response) { + if resp.StatusCode == 403 { + if self.cpcfg.UpdatePermission != nil { + self.cpcfg.UpdatePermission(service, fmt.Sprintf("%s %s", method, path)) + } + } + } + if self.cpcfg.ReadOnly { + if req.Method == "GET" { + return respCheck, nil + } + return nil, errors.Wrapf(cloudprovider.ErrAccountReadOnly, "%s %s", req.Method, req.URL.Path) + } + return respCheck, nil + }) + return self.httpClient +} + +func (self *akClient) Do(req *http.Request) (*http.Response, error) { + req.Header.Del("Accept") + if req.Method == string(httputils.GET) || req.Method == string(httputils.DELETE) || req.Method == string(httputils.PATCH) { + req.Header.Del("Content-Length") + } + err := self.signer.Sign(req) + if err != nil { + return nil, errors.Wrapf(err, "Sign") + } + return self.client.Do(req) +} + +func (self *SHcsClient) getAkClient() *akClient { + return &akClient{ + client: self.getDefaultClient(), + signer: &Signer{ + Key: self.accessKey, + Secret: self.accessSecret, + }, + } +} + +type hcsError struct { + Url string + Params map[string]interface{} + Code int `json:"code,omitzero"` + Class string `json:"class,omitempty"` + ErrorMsg string `json:"error_msg,omitempty"` + Details string `json:"details,omitempty"` +} + +func (ce *hcsError) Error() string { + return jsonutils.Marshal(ce).String() +} + +func (ce *hcsError) ParseErrorFromJsonResponse(statusCode int, body jsonutils.JSONObject) error { + body.Unmarshal(ce) + if ce.Code == 0 { + ce.Code = statusCode + } + if len(ce.Class) == 0 { + ce.Class = http.StatusText(statusCode) + } + if len(ce.Details) == 0 { + ce.Details = body.String() + } + return ce +} + +type sThrottlingThreshold struct { + locked bool + lockTime time.Time +} + +func (t *sThrottlingThreshold) CheckingLock() { + if !t.locked { + return + } + + for { + if t.lockTime.Sub(time.Now()).Seconds() < 0 { + return + } + log.Debugf("throttling threshold has been reached. release at %s", t.lockTime) + time.Sleep(5 * time.Second) + } +} + +func (t *sThrottlingThreshold) Lock() { + // 锁定至少15秒 + t.locked = true + t.lockTime = time.Now().Add(15 * time.Second) +} + +func (self *SHcsClient) request(method httputils.THttpMethod, url string, query url.Values, params map[string]interface{}) (jsonutils.JSONObject, error) { + self.lock.CheckingLock() + client := self.getAkClient() + if len(query) > 0 { + url = fmt.Sprintf("%s?%s", url, query.Encode()) + } + url = strings.TrimPrefix(url, "http://") + if !strings.HasPrefix(url, "https://") { + url = fmt.Sprintf("https://%s", url) + } + var body jsonutils.JSONObject = nil + if len(params) > 0 { + body = jsonutils.Marshal(params) + } + header := http.Header{} + if len(self.projectId) > 0 && !strings.Contains(url, "v3/regions") { + header.Set("X-Project-Id", self.projectId) + } + if len(self.domainId) > 0 { + //header.Set("X-Domain-Id", self.domainId) + } + cli := httputils.NewJsonClient(client) + req := httputils.NewJsonRequest(method, url, body) + req.SetHeader(header) + var resp jsonutils.JSONObject + var err error + for i := 0; i < 4; i++ { + _, resp, err = cli.Send(context.Background(), req, &hcsError{Url: url, Params: params}, self.debug) + if err == nil { + break + } + if err != nil { + e, ok := err.(*hcsError) + if ok { + if e.Code == 404 { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, err.Error()) + } + if e.Code == 429 { + log.Errorf("request %s %v try later", url, err) + self.lock.Lock() + time.Sleep(time.Second * 15) + continue + } + } + return nil, err + } + } + return resp, err +} + +func (self *SHcsClient) iamGet(resource string, query url.Values) (jsonutils.JSONObject, error) { + url := fmt.Sprintf("iam-apigateway-proxy.%s/%s", self.authUrl, resource) + return self.request(httputils.GET, url, query, nil) +} + +func (self *SHcsClient) fetchRegions() error { + if len(self.regions) > 0 { + return nil + } + resp, err := self.iamGet("v3/regions", nil) + if err != nil { + return err + } + self.regions = []SRegion{} + err = resp.Unmarshal(&self.regions, "regions") + if err != nil { + return err + } + for i := range self.regions { + self.defaultRegion = self.regions[i].Id + } + return nil +} + +func (hcscli *SHcsClient) rdsList(regionId string, resource string, query url.Values, retVal interface{}) error { + return hcscli.list("rds", "v3", regionId, resource, query, retVal) +} + +func (hcscli *SHcsClient) rdsGet(regionId string, resource string, retVal interface{}) error { + return hcscli.get("rds", "v3", regionId, resource, retVal) +} + +func (hcscli *SHcsClient) rdsDelete(regionId string, resource string) error { + return hcscli._delete("rds", "v3", regionId, resource) +} + +func (hcscli *SHcsClient) rdsCreate(regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + return hcscli._create("rds", "v3", regionId, resource, body, retVal) +} + +func (self *SHcsClient) rdsPerform(regionId string, resource, action string, params map[string]interface{}, retVal interface{}) error { + return self._perform("rds", "v3", regionId, resource, action, params, retVal) +} + +func (hcscli *SHcsClient) rdsJobGet(regionId string, resource string, query url.Values, retVal interface{}) error { + url := hcscli._url("rds", "v3", regionId, resource) + resp, err := hcscli.request(httputils.GET, url, query, nil) + if err != nil { + return err + } + err = resp.Unmarshal(retVal) + return err +} + +func (hcscli *SHcsClient) rdsDBPrivvilegesDelete(regionId string, resource string, params map[string]interface{}) error { + return hcscli._deleteWithBody("rds", "v3", regionId, resource, params) +} + +func (hcscli *SHcsClient) rdsDBPrivilegesGrant(regionId string, resource string, params map[string]interface{}, retVal interface{}) error { + url := hcscli._url("rds", "v3", regionId, resource) + resp, err := hcscli.request(httputils.GET, url, nil, params) + if err != nil { + return err + } + err = resp.Unmarshal(retVal) + return err +} + +func (self *SHcsClient) ecsList(regionId string, resource string, query url.Values, retVal interface{}) error { + return self.list("ecs", "v2", regionId, resource, query, retVal) +} + +func (self *SHcsClient) list(product, version, regionId string, resource string, query url.Values, retVal interface{}) error { + resp, err := self._list(product, version, regionId, resource, query) + if err != nil { + return errors.Wrapf(err, "_list") + } + return resp.Unmarshal(retVal) +} + +func (self *SHcsClient) ecsGet(regionId string, resource string, retVal interface{}) error { + return self.get("ecs", "v2", regionId, resource, retVal) +} + +func (self *SHcsClient) ecsDelete(regionId string, resource string) error { + return self._delete("ecs", "v2", regionId, resource) +} + +func (self *SHcsClient) ecsPerform(regionId string, resource, action string, params map[string]interface{}, retVal interface{}) error { + return self._perform("ecs", "v2", regionId, resource, action, params, retVal) +} + +func (self *SHcsClient) evsDelete(regionId string, resource string) error { + return self._delete("evs", "v2", regionId, resource) +} + +func (self *SHcsClient) evsList(regionId string, resource string, query url.Values, retVal interface{}) error { + return self.list("evs", "v2", regionId, resource, query, retVal) +} + +func (self *SHcsClient) evsGet(regionId string, resource string, retVal interface{}) error { + return self.get("evs", "v2", regionId, resource, retVal) +} + +func (self *SHcsClient) evsPerform(regionId string, resource, action string, params map[string]interface{}) error { + return self._perform("evs", "v2", regionId, resource, action, params, nil) +} + +func (self *SHcsClient) ecsCreate(regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + return self._create("ecs", "v2", regionId, resource, body, retVal) +} + +func (self *SHcsClient) evsCreate(regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + return self._create("evs", "v2", regionId, resource, body, retVal) +} + +func (self *SHcsClient) vpcCreate(regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + return self._create("vpc", "v1", regionId, resource, body, retVal) +} + +func (self *SHcsClient) vpcDelete(regionId string, resource string) error { + return self._delete("vpc", "v1", regionId, resource) +} + +func (self *SHcsClient) vpcGet(regionId string, resource string, retVal interface{}) error { + return self.get("vpc", "v1", regionId, resource, retVal) +} + +func (self *SHcsClient) vpcList(regionId string, resource string, query url.Values, retVal interface{}) error { + return self.list("vpc", "v1", regionId, resource, query, retVal) +} + +func (self *SHcsClient) vpcUpdate(regionId string, resource string, body map[string]interface{}) error { + return self._update("vpc", "v1", regionId, resource, body) +} + +func (self *SHcsClient) imsCreate(regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + return self._create("ims", "v2", regionId, resource, body, retVal) +} + +func (self *SHcsClient) imsDelete(regionId string, resource string) error { + return self._delete("ims", "v2", regionId, resource) +} + +func (self *SHcsClient) imsGet(regionId string, resource string, retVal interface{}) error { + return self.get("ims", "v2", regionId, resource, retVal) +} + +func (self *SHcsClient) imsList(regionId string, resource string, query url.Values, retVal interface{}) error { + return self.list("ims", "v2", regionId, resource, query, retVal) +} + +func (self *SHcsClient) imsPerform(regionId string, resource, action string, params map[string]interface{}, retVal interface{}) error { + return self._perform("ims", "v2", regionId, resource, action, params, retVal) +} + +func (self *SHcsClient) imsUpdate(regionId string, resource string, body map[string]interface{}) error { + return self._update("ims", "v2", regionId, resource, body) +} + +func (self *SHcsClient) dcsCreate(regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + return self._create("dcs", "v1.0", regionId, resource, body, retVal) +} + +func (self *SHcsClient) dcsDelete(regionId string, resource string) error { + return self._delete("dcs", "v1.0", regionId, resource) +} + +func (self *SHcsClient) dcsGet(regionId string, resource string, retVal interface{}) error { + return self.get("dcs", "v1.0", regionId, resource, retVal) +} + +func (self *SHcsClient) dcsList(regionId string, resource string, query url.Values, retVal interface{}) error { + return self.list("dcs", "v1.0", regionId, resource, query, retVal) +} + +func (self *SHcsClient) dcsPerform(regionId string, resource, action string, params map[string]interface{}, retVal interface{}) error { + return self._perform("dcs", "v1.0", regionId, resource, action, params, retVal) +} + +func (self *SHcsClient) dcsUpdate(regionId string, resource string, body map[string]interface{}) error { + return self._update("dcs", "v1.0", regionId, resource, body) +} + +func (self *SHcsClient) _url(product, version, regionId string, resource string) string { + url := fmt.Sprintf("%s.%s.%s/%s/%s/%s", product, regionId, self.authUrl, version, self.projectId, resource) + for _, prefix := range []string{ + "images", "cloudimages", "nat_gateways", + "lbaas", "products", "snat_rules", + "dnat_rules", "vpc/peerings", + } { + if strings.HasPrefix(resource, prefix) { + url = fmt.Sprintf("%s.%s.%s/%s/%s", product, regionId, self.authUrl, version, resource) + break + } + } + return url +} + +func (self *SHcsClient) _list(product, version, regionId string, resource string, query url.Values) (*jsonutils.JSONArray, error) { + ret := jsonutils.NewArray() + url := self._url(product, version, regionId, resource) + offset, _ := strconv.Atoi(query.Get("offset")) + total := int64(0) + for { + resp, err := self.request(httputils.GET, url, query, nil) + if err != nil { + return nil, err + } + if gotypes.IsNil(resp) { + log.Warningf("%s return empty", resource) + return ret, nil + } + objMap, err := resp.GetMap() + if err != nil { + return nil, errors.Wrapf(err, "resp.GetMap") + } + next := false + for k, v := range objMap { + if k == "links" { + url, _ = v.GetString("next") + next = true + continue + } + if k == "count" { + offset++ + query.Set("offset", fmt.Sprintf("%d", offset)) + total, _ = v.Int() + next = true + continue + } + if strings.Contains(resource, k) || + strings.Contains(strings.ReplaceAll(resource, "-", "_"), k) || + utils.IsInStringArray(k, []string{ + "availabilityZoneInfo", + }) { + objs, err := v.GetArray() + if err != nil { + return nil, errors.Wrapf(err, "v.GetArray") + } + ret.Add(objs...) + } + } + if !next || len(url) == 0 || (ret.Length() == int(total)) { + break + } + } + return ret, nil +} + +func (self *SHcsClient) get(product, version, regionId string, resource string, retVal interface{}) error { + resp, err := self._get(product, version, regionId, resource) + if err != nil { + return err + } + obj, err := resp.GetMap() + if err != nil { + return errors.Wrapf(err, "GetMap") + } + for v := range obj { + if len(obj) == 1 { + return obj[v].Unmarshal(retVal) + } + } + return resp.Unmarshal(retVal) +} + +func (self *SHcsClient) _getJob(product, regionId string, jobId string) (jsonutils.JSONObject, error) { + url := self._url(product, "v1", regionId, fmt.Sprintf("jobs/%s", jobId)) + resp, err := self.request(httputils.GET, url, nil, nil) + if err != nil { + return nil, err + } + return resp, nil +} + +func (self *SHcsClient) _get(product, version, regionId string, resource string) (jsonutils.JSONObject, error) { + url := self._url(product, version, regionId, resource) + resp, err := self.request(httputils.GET, url, nil, nil) + if err != nil { + return nil, err + } + return resp, nil +} + +func (self *SHcsClient) delete(product, version, regionId string, resource string) error { + return self._delete(product, version, regionId, resource) +} + +func (self *SHcsClient) _delete(product, version, regionId string, resource string) error { + url := self._url(product, version, regionId, resource) + resp, err := self.request(httputils.DELETE, url, nil, nil) + if !gotypes.IsNil(resp) && resp.Contains("job_id") { + jobId, _ := resp.GetString("job_id") + _, err := self.waitJobSuccess(product, regionId, jobId, time.Second*10, time.Hour*2) + if err != nil { + return errors.Wrapf(err, "wait create %s %s job", product, resource) + } + return nil + } + return err +} + +func (self *SHcsClient) _deleteWithBody(product, version, regionId string, resource string, params map[string]interface{}) error { + url := self._url(product, version, regionId, resource) + _, err := self.request(httputils.DELETE, url, nil, params) + return err +} + +func (self *SHcsClient) update(product, version, regionId string, resource string, params map[string]interface{}) error { + return self._update(product, version, regionId, resource, params) +} + +func (self *SHcsClient) perform(product, version, regionId string, resource, action string, params map[string]interface{}, retVal interface{}) error { + return self._perform(product, version, regionId, resource, action, params, retVal) +} + +func (self *SHcsClient) _perform(product, version, regionId string, resource, action string, params map[string]interface{}, retVal interface{}) error { + url := self._url(product, version, regionId, resource+"/"+action) + resp, err := self.request(httputils.POST, url, nil, params) + if err != nil { + return err + } + if !gotypes.IsNil(resp) && resp.Contains("job_id") { + jobId, _ := resp.GetString("job_id") + job, err := self.waitJobSuccess(product, regionId, jobId, time.Second*10, time.Hour*2) + if err != nil { + return errors.Wrapf(err, "wait create %s %s job", product, resource) + } + if retVal != nil { + return jsonutils.Update(retVal, jsonutils.Marshal(job)) + } + return nil + } + if retVal != nil { + return resp.Unmarshal(retVal) + } + return nil +} + +func (self *SHcsClient) create(product, version, regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + return self._create(product, version, regionId, resource, body, retVal) +} + +func (self *SHcsClient) _create(product, version, regionId string, resource string, body map[string]interface{}, retVal interface{}) error { + url := self._url(product, version, regionId, resource) + resp, err := self.request(httputils.POST, url, nil, body) + if err != nil { + return err + } + if resp.Contains("job_id") { + jobId, _ := resp.GetString("job_id") + job, err := self.waitJobSuccess(product, regionId, jobId, time.Second*10, time.Hour*2) + if err != nil { + return errors.Wrapf(err, "wait create %s %s job", product, resource) + } + if retVal != nil { + return jsonutils.Update(retVal, jsonutils.Marshal(job)) + } + return nil + } + obj, err := resp.GetMap() + if err != nil { + return errors.Wrapf(err, "GetMap") + } + for v := range obj { + if len(obj) == 1 && retVal != nil { + return obj[v].Unmarshal(retVal) + } + } + if retVal != nil { + return resp.Unmarshal(retVal) + } + return nil +} + +func (self *SHcsClient) _update(product, version, regionId string, resource string, body map[string]interface{}) error { + url := self._url(product, version, regionId, resource) + _, err := self.request(httputils.PUT, url, nil, body) + return err +} + +func (self *SHcsClient) GetRegions() []SRegion { + return self.regions +} + +func (self *SHcsClient) GetIRegions() []cloudprovider.ICloudRegion { + ret := []cloudprovider.ICloudRegion{} + if len(self.projectId) == 0 { + for i := range self.regions { + self.regions[i].client = self + self.defaultRegion = self.regions[i].Id + ret = append(ret, &self.regions[i]) + } + return ret + } else { + for i := range self.regions { + if strings.Contains(self.projectName, self.regions[i].Id) { + self.defaultRegion = self.regions[i].Id + self.regions[i].client = self + ret = append(ret, &self.regions[i]) + } + if self.projectName == self.regions[i].Id { + self.isMainProject = true + } + } + } + return ret +} + +func (self *SHcsClient) GetCapabilities() []string { + caps := []string{ + cloudprovider.CLOUD_CAPABILITY_PROJECT, + cloudprovider.CLOUD_CAPABILITY_COMPUTE, + cloudprovider.CLOUD_CAPABILITY_NETWORK, + cloudprovider.CLOUD_CAPABILITY_CACHE, + cloudprovider.CLOUD_CAPABILITY_RDS, + cloudprovider.CLOUD_CAPABILITY_LOADBALANCER, + cloudprovider.CLOUD_CAPABILITY_NAT, + } + // huawei objectstore is shared across projects(subscriptions) + // to avoid multiple project access the same bucket + // only main project is allow to access objectstore bucket + if self.isMainProject { + caps = append(caps, cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE) + } + return caps +} + +func (self *SHcsClient) getOBSEndpoint(regionId string) string { + return fmt.Sprintf("obs.%s.%s", regionId, self.authUrl) +} + +func (self *SHcsClient) getOBSClient(regionId string) (*obs.ObsClient, error) { + endpoint := self.getOBSEndpoint(regionId) + ts, _ := self.httpClient.Transport.(*http.Transport) + conf := obs.WithHttpTransport(ts) + cli, err := obs.New(self.accessKey, self.accessSecret, endpoint, conf) + if err != nil { + return nil, err + } + return cli, nil +} + +func (self *SHcsClient) GetBuckets() ([]SBucket, error) { + if len(self.buckets) > 0 { + return self.buckets, nil + } + obscli, err := self.getOBSClient(self.regions[0].Id) + if err != nil { + return nil, errors.Wrap(err, "getOBSClient") + } + input := &obs.ListBucketsInput{QueryLocation: true} + output, err := obscli.ListBuckets(input) + if err != nil { + return nil, errors.Wrap(err, "obscli.ListBuckets") + } + self.buckets = []SBucket{} + for i := range output.Buckets { + bInfo := output.Buckets[i] + region, err := self.GetRegion(bInfo.Location) + if err != nil { + log.Errorf("fail to find region %s", bInfo.Location) + continue + } + b := SBucket{ + region: region, + + Name: bInfo.Name, + Location: bInfo.Location, + CreationDate: bInfo.CreationDate, + } + self.buckets = append(self.buckets, b) + } + return self.buckets, nil +} diff --git a/pkg/multicloud/hcs/host.go b/pkg/multicloud/hcs/host.go new file mode 100644 index 0000000000..a0d367bd9e --- /dev/null +++ b/pkg/multicloud/hcs/host.go @@ -0,0 +1,254 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/osprofile" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/billing" +) + +type SHost struct { + multicloud.SHostBase + zone *SZone +} + +func (self *SHost) GetId() string { + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) +} + +func (self *SHost) GetName() string { + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId()) +} + +func (self *SHost) GetGlobalId() string { + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) +} + +func (self *SHost) IsEmulated() bool { + return true +} + +func (self *SHost) GetStatus() string { + return api.HOST_STATUS_RUNNING +} + +func (self *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) { + vms, err := self.zone.region.GetInstances("") + if err != nil { + return nil, err + } + + ret := []cloudprovider.ICloudVM{} + for i := range vms { + if vms[i].OSEXTAZAvailabilityZone == self.zone.GetId() { + vms[i].host = self + ret = append(ret, &vms[i]) + } + } + return ret, nil +} + +func (self *SHost) GetIVMById(id string) (cloudprovider.ICloudVM, error) { + vm, err := self.zone.region.GetInstance(id) + if err != nil { + return nil, err + } + vm.host = self + return vm, nil +} + +func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { + return self.zone.GetIWires() +} + +func (self *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + return self.zone.GetIStorages() +} + +func (self *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + return self.zone.GetIStorageById(id) +} + +func (self *SHost) GetEnabled() bool { + return true +} + +func (self *SHost) GetHostStatus() string { + return api.HOST_ONLINE +} + +func (self *SHost) GetAccessIp() string { + return "" +} + +func (self *SHost) GetAccessMac() string { + return "" +} + +func (self *SHost) GetSysInfo() jsonutils.JSONObject { + info := jsonutils.NewDict() + info.Add(jsonutils.NewString(CLOUD_PROVIDER_HCS), "manufacture") + return info +} + +func (self *SHost) GetSN() string { + return "" +} + +func (self *SHost) GetCpuCount() int { + return 0 +} + +func (self *SHost) GetNodeCount() int8 { + return 0 +} + +func (self *SHost) GetCpuDesc() string { + return "" +} + +func (self *SHost) GetCpuMhz() int { + return 0 +} + +func (self *SHost) GetMemSizeMB() int { + return 0 +} + +func (self *SHost) GetStorageSizeMB() int { + return 0 +} + +func (self *SHost) GetStorageType() string { + return api.DISK_TYPE_HYBRID +} + +func (self *SHost) GetHostType() string { + return api.HOST_TYPE_HCS +} + +func (self *SHost) GetIsMaintenance() bool { + return false +} + +func (self *SHost) GetVersion() string { + return HCS_API_VERSION +} + +func (self *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) { + vm, err := self._createVM( + desc.Name, desc.ExternalImageId, desc.SysDisk, + desc.Cpu, desc.MemoryMB, desc.InstanceType, + desc.ExternalNetworkId, desc.IpAddr, + desc.Description, desc.Account, + desc.Password, desc.DataDisks, + desc.PublicKey, desc.ExternalSecgroupId, + desc.UserData, desc.BillingCycle, desc.ProjectId, desc.Tags) + if err != nil { + return nil, err + } + return vm, err +} + +func (self *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (self *SHost) _createVM(name string, imgId string, sysDisk cloudprovider.SDiskInfo, cpu int, memMB int, instanceType string, + networkId string, ipAddr string, desc string, account string, passwd string, + diskSizes []cloudprovider.SDiskInfo, publicKey string, secgroupId string, + userData string, bc *billing.SBillingCycle, projectId string, tags map[string]string) (*SInstance, error) { + net, err := self.zone.region.GetNetwork(networkId) + if err != nil { + return nil, errors.Wrapf(err, "GetNetwork(%s)", networkId) + } + + keypair := "" + if len(publicKey) > 0 { + keypair, err = self.zone.region.syncKeypair(publicKey) + if err != nil { + return nil, err + } + } + + // 镜像及硬盘配置 + img, err := self.zone.region.GetImage(imgId) + if err != nil { + return nil, errors.Wrapf(err, "GetImage(%s)", imgId) + } + // passwd, windows机型直接使用密码比较方便 + if strings.ToLower(img.Platform) == strings.ToLower(osprofile.OS_TYPE_WINDOWS) && len(passwd) > 0 { + keypair = "" + } + + if strings.ToLower(img.Platform) == strings.ToLower(osprofile.OS_TYPE_WINDOWS) { + if u, err := updateWindowsUserData(userData, img.OSVersion, account, passwd); err == nil { + userData = u + } else { + return nil, errors.Wrap(err, "SHost.CreateVM.updateWindowsUserData") + } + } + + if sysDisk.SizeGB > 0 && sysDisk.SizeGB < img.SizeGB { + sysDisk.SizeGB = img.SizeGB + } + + // 创建实例 + if len(instanceType) > 0 { + log.Debugf("Try instancetype : %s", instanceType) + vm, err := self.zone.region.CreateInstance(name, imgId, instanceType, networkId, secgroupId, + net.VpcId, self.zone.GetId(), desc, sysDisk, diskSizes, ipAddr, keypair, + passwd, userData, bc, projectId, tags) + if err != nil { + return nil, fmt.Errorf("create %s failed:%v", instanceType, err) + } + return vm, nil + } + + // 匹配实例类型 + instanceTypes, err := self.zone.region.GetMatchInstanceTypes(cpu, memMB, self.zone.GetId()) + if err != nil { + return nil, err + } + if len(instanceTypes) == 0 { + return nil, fmt.Errorf("instance type %dC%dMB not avaiable", cpu, memMB) + } + + var vm *SInstance + for _, instType := range instanceTypes { + instanceTypeId := instType.Name + log.Debugf("Try instancetype : %s", instanceTypeId) + vm, err = self.zone.region.CreateInstance(name, imgId, instanceTypeId, networkId, secgroupId, + net.VpcId, self.zone.GetId(), desc, sysDisk, diskSizes, ipAddr, keypair, + passwd, userData, bc, projectId, tags) + if err != nil { + log.Errorf("Failed for %s: %s", instanceTypeId, err) + } else { + return vm, nil + } + } + + return nil, fmt.Errorf("create failed: %v", err) +} diff --git a/pkg/multicloud/hcs/image.go b/pkg/multicloud/hcs/image.go new file mode 100644 index 0000000000..ed8bdeff10 --- /dev/null +++ b/pkg/multicloud/hcs/image.go @@ -0,0 +1,466 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/imagetools" +) + +type TImageOwnerType string + +const ( + ImageOwnerPublic TImageOwnerType = "gold" // 公共镜像：gold + ImageOwnerSelf TImageOwnerType = "private" // 私有镜像：private + ImageOwnerShared TImageOwnerType = "shared" // 共享镜像：shared + + EnvFusionCompute = "FusionCompute" + EnvIronic = "Ironic" +) + +const ( + ImageStatusQueued = "queued" // queued：表示镜像元数据已经创建成功，等待上传镜像文件。 + ImageStatusSaving = "saving" // saving：表示镜像正在上传文件到后端存储。 + ImageStatusDeleted = "deleted" // deleted：表示镜像已经删除。 + ImageStatusKilled = "killed" // killed：表示镜像上传错误。 + ImageStatusActive = "active" // active：表示镜像可以正常使用 +) + +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0020091565.html +type SImage struct { + multicloud.SImageBase + multicloud.HuaweiTags + cache *SStoragecache + + // normalized image info + imgInfo *imagetools.ImageInfo + + Schema string `json:"schema"` + MinDiskGB int64 `json:"min_disk"` + CreatedAt time.Time `json:"created_at"` + ImageSourceType string `json:"__image_source_type"` + ContainerFormat string `json:"container_format"` + File string `json:"file"` + UpdatedAt time.Time `json:"updated_at"` + Protected bool `json:"protected"` + Checksum string `json:"checksum"` + Id string `json:"id"` + Isregistered string `json:"__isregistered"` + MinRamMB int `json:"min_ram"` + Lazyloading string `json:"__lazyloading"` + Owner string `json:"owner"` + OSType string `json:"__os_type"` + Imagetype string `json:"__imagetype"` + Visibility string `json:"visibility"` + VirtualEnvType string `json:"virtual_env_type"` + Platform string `json:"__platform"` + SizeGB int `json:"size"` + ImageSize int64 `json:"__image_size"` + OSBit string `json:"__os_bit"` + OSVersion string `json:"__os_version"` + Architecture string + Name string `json:"name"` + Self string `json:"self"` + DiskFormat string `json:"disk_format"` + Status string `json:"status"` + SupportKVMFPGAType string `json:"__support_kvm_fpga_type"` + SupportKVMNVMEHIGHIO string `json:"__support_nvme_highio"` + SupportLargeMemory string `json:"__support_largememory"` + SupportDiskIntensive string `json:"__support_diskintensive"` + SupportHighPerformance string `json:"__support_highperformance"` + SupportXENGPUType string `json:"__support_xen_gpu_type"` + SupportKVMGPUType string `json:"__support_kvm_gpu_type"` + SupportGPUT4 string `json:"__support_gpu_t4"` + SupportKVMAscend310 string `json:"__support_kvm_ascend_310"` + SupportArm string `json:"__support_arm"` +} + +func (self *SImage) GetMinRamSizeMb() int { + return self.MinRamMB +} + +func (self *SImage) GetId() string { + return self.Id +} + +func (self *SImage) GetName() string { + return self.Name +} + +func (self *SImage) GetGlobalId() string { + return self.Id +} + +func (self *SImage) GetStatus() string { + switch self.Status { + case ImageStatusQueued: + return api.CACHED_IMAGE_STATUS_CACHING + case ImageStatusActive: + return api.CACHED_IMAGE_STATUS_ACTIVE + case ImageStatusKilled: + return api.CACHED_IMAGE_STATUS_CACHE_FAILED + default: + return api.CACHED_IMAGE_STATUS_CACHE_FAILED + } +} + +func (self *SImage) GetImageStatus() string { + switch self.Status { + case ImageStatusQueued: + return cloudprovider.IMAGE_STATUS_QUEUED + case ImageStatusActive: + return cloudprovider.IMAGE_STATUS_ACTIVE + case ImageStatusKilled: + return cloudprovider.IMAGE_STATUS_KILLED + default: + return cloudprovider.IMAGE_STATUS_KILLED + } +} + +func (self *SImage) Refresh() error { + ret, err := self.cache.region.GetImage(self.GetId()) + if err != nil { + return err + } + return jsonutils.Update(self, ret) +} + +func (self *SImage) GetImageType() cloudprovider.TImageType { + switch self.Imagetype { + case "gold": + return cloudprovider.ImageTypeSystem + case "private": + return cloudprovider.ImageTypeCustomized + case "shared": + return cloudprovider.ImageTypeShared + default: + return cloudprovider.ImageTypeCustomized + } +} + +func (self *SImage) GetSizeByte() int64 { + return int64(self.MinDiskGB) * 1024 * 1024 * 1024 +} + +func (self *SImage) getNormalizedImageInfo() *imagetools.ImageInfo { + if self.imgInfo == nil { + imgInfo := imagetools.NormalizeImageInfo(self.ImageSourceType, self.Architecture, self.OSType, self.Platform, "") + self.imgInfo = &imgInfo + } + + return self.imgInfo +} + +func (self *SImage) GetFullOsName() string { + return self.ImageSourceType +} + +func (self *SImage) GetOsType() cloudprovider.TOsType { + return cloudprovider.TOsType(self.getNormalizedImageInfo().OsType) +} + +func (self *SImage) GetOsDist() string { + return self.getNormalizedImageInfo().OsDistro +} + +func (self *SImage) GetOsVersion() string { + return self.getNormalizedImageInfo().OsVersion +} + +func (self *SImage) GetOsLang() string { + return self.getNormalizedImageInfo().OsLang +} + +func (self *SImage) GetOsArch() string { + return self.getNormalizedImageInfo().OsArch +} + +func (self *SImage) GetBios() cloudprovider.TBiosType { + return cloudprovider.ToBiosType(self.getNormalizedImageInfo().OsBios) +} + +func (self *SImage) GetMinOsDiskSizeGb() int { + return int(self.MinDiskGB) +} + +func (self *SImage) GetImageFormat() string { + return self.DiskFormat +} + +func (self *SImage) GetCreatedAt() time.Time { + return self.CreatedAt +} + +func (self *SImage) IsEmulated() bool { + return false +} + +func (self *SImage) Delete(ctx context.Context) error { + return self.cache.region.DeleteImage(self.GetId()) +} + +func (self *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache { + return self.cache +} + +func (self *SRegion) GetImage(imageId string) (*SImage, error) { + image := &SImage{cache: self.getStoragecache()} + res := fmt.Sprintf("images/%s", imageId) + return image, self.imsGet(res, image) +} + +func excludeImage(image SImage) bool { + if image.VirtualEnvType == "Ironic" { + return true + } + + if len(image.SupportDiskIntensive) > 0 { + return true + } + + if len(image.SupportKVMFPGAType) > 0 || len(image.SupportKVMAscend310) > 0 { + return true + } + + if len(image.SupportKVMGPUType) > 0 { + return true + } + + if len(image.SupportKVMNVMEHIGHIO) > 0 { + return true + } + + if len(image.SupportGPUT4) > 0 { + return true + } + + if len(image.SupportXENGPUType) > 0 { + return true + } + + if len(image.SupportHighPerformance) > 0 { + return true + } + + if len(image.SupportArm) > 0 { + return true + } + + return false +} + +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0060804959.html +func (self *SRegion) GetImages(status string, imagetype TImageOwnerType, name string, envType string) ([]SImage, error) { + params := url.Values{} + if len(status) > 0 { + params.Set("status", status) + } + + if len(imagetype) > 0 { + params.Set("__imagetype", string(imagetype)) + if imagetype == ImageOwnerPublic { + params.Set("protected", "True") + } + } + if len(envType) > 0 { + params.Set("virtual_env_type", envType) + } + + if len(name) > 0 { + params.Set("name", name) + } + + images := []SImage{} + err := self.imsList("images", params, &images) + if err != nil { + return nil, err + } + + // 排除掉需要特定镜像才能创建的实例类型 + // https://support.huaweicloud.com/eu-west-0-api-ims/zh-cn_topic_0031617666.html#ZH-CN_TOPIC_0031617666__table48545918250 + // https://support.huaweicloud.com/productdesc-ecs/zh-cn_topic_0088142947.html + filtedImages := make([]SImage, 0) + for i := range images { + if !excludeImage(images[i]) { + filtedImages = append(filtedImages, images[i]) + } + } + return filtedImages, err +} + +func (self *SRegion) DeleteImage(imageId string) error { + res := fmt.Sprintf("images/%s", imageId) + return self.imsDelete(res) +} + +func (self *SRegion) GetImageByName(name string) (*SImage, error) { + if len(name) == 0 { + return nil, fmt.Errorf("image name should not be empty") + } + images, err := self.GetImages("", TImageOwnerType(""), name, "") + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, cloudprovider.ErrNotFound + } + log.Debugf("%d image found match name %s", len(images), name) + return &images[0], nil +} + +/* https://support.huaweicloud.com/api-ims/zh-cn_topic_0020092109.html + os version 取值范围： https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html + 用于创建私有镜像的源云服务器系统盘大小大于等于40GB且不超过1024GB。 + 目前支持vhd，zvhd、raw，qcow2 + todo: 考虑使用镜像快速导入。 https://support.huaweicloud.com/api-ims/zh-cn_topic_0133188204.html + 使用OBS文件创建镜像 + + * openstack原生接口支持的格式：https://support.huaweicloud.com/api-ims/zh-cn_topic_0031615566.html +*/ +func (self *SRegion) ImportImageJob(name string, osDist string, osVersion string, osArch string, bucket string, key string, minDiskGB int64) error { + os_version, err := stdVersion(osDist, osVersion, osArch) + log.Debugf("%s %s %s: %s.min_disk %d GB", osDist, osVersion, osArch, os_version, minDiskGB) + if err != nil { + log.Debugln(err) + } + + image_url := fmt.Sprintf("%s:%s", bucket, key) + params := map[string]interface{}{ + "name": name, + "image_url": image_url, + "os_version": os_version, + "is_config_init": true, + "is_config": true, + "min_disk": minDiskGB, + } + return self.imsPerform("cloudimage", "action", params, nil) +} + +func formatVersion(osDist string, osVersion string) (string, error) { + err := fmt.Errorf("unsupport version %s.reference: https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html", osVersion) + dist := strings.ToLower(osDist) + if dist == "ubuntu" || dist == "redhat" || dist == "centos" || dist == "oracle" || dist == "euleros" { + parts := strings.Split(osVersion, ".") + if len(parts) < 2 { + return "", err + } + + return parts[0] + "." + parts[1], nil + } + + if dist == "debian" { + parts := strings.Split(osVersion, ".") + if len(parts) < 3 { + return "", err + } + + return parts[0] + "." + parts[1] + "." + parts[2], nil + } + + if dist == "fedora" || dist == "windows" || dist == "suse" { + parts := strings.Split(osVersion, ".") + if len(parts) < 1 { + return "", err + } + + return parts[0], nil + } + + if dist == "opensuse" { + parts := strings.Split(osVersion, ".") + if len(parts) == 0 { + return "", err + } + + if len(parts) == 1 { + return parts[0], nil + } + + if len(parts) >= 2 { + return parts[0] + "." + parts[1], nil + } + } + + return "", err +} + +// todo: 如何保持同步更新 +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html +func stdVersion(osDist string, osVersion string, osArch string) (string, error) { + // 架构 + arch := "" + switch osArch { + case "64", apis.OS_ARCH_X86_64: + arch = "64bit" + case "32", apis.OS_ARCH_X86_32: + arch = "32bit" + default: + return "", fmt.Errorf("unsupported arch %s.reference: https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html", osArch) + } + + _dist := strings.Split(strings.TrimSpace(osDist), " ")[0] + _dist = strings.ToLower(_dist) + // 版本 + ver, err := formatVersion(_dist, osVersion) + if err != nil { + return "", err + } + + // 操作系统 + dist := "" + + switch _dist { + case "ubuntu": + return fmt.Sprintf("Ubuntu %s server %s", ver, arch), nil + case "redhat": + dist = "Redhat Linux Enterprise" + case "centos": + dist = "CentOS" + case "fedora": + dist = "Fedora" + case "debian": + dist = "Debian GNU/Linux" + case "windows": + dist = "Windows Server" + case "oracle": + dist = "Oracle Linux Server release" + case "suse": + dist = "SUSE Linux Enterprise Server" + case "opensuse": + dist = "OpenSUSE" + case "euleros": + dist = "EulerOS" + default: + return "", fmt.Errorf("unsupported os %s. reference: https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html", dist) + } + + return fmt.Sprintf("%s %s %s", dist, ver, arch), nil +} + +func (self *SImage) UEFI() bool { + return false +} diff --git a/pkg/multicloud/hcs/instance.go b/pkg/multicloud/hcs/instance.go new file mode 100644 index 0000000000..1f0951c732 --- /dev/null +++ b/pkg/multicloud/hcs/instance.go @@ -0,0 +1,1114 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/apis" + billing_api "yunion.io/x/onecloud/pkg/apis/billing" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/cloudinit" + "yunion.io/x/onecloud/pkg/util/imagetools" +) + +const ( + InstanceStatusRunning = "ACTIVE" + InstanceStatusTerminated = "DELETED" + InstanceStatusStopped = "SHUTOFF" +) + +type IpAddress struct { + Version string `json:"version"` + Addr string `json:"addr"` + OSEXTIPSMACMACAddr string `json:"OS-EXT-IPS-MAC:mac_addr"` + OSEXTIPSPortId string `json:"OS-EXT-IPS:port_id"` + OSEXTIPSType string `json:"OS-EXT-IPS:type"` +} + +type Flavor struct { + Disk string `json:"disk"` + Vcpus string `json:"vcpus"` + RAM string `json:"ram"` + Id string `json:"id"` + Name string `json:"name"` +} + +type Image struct { + Id string `json:"id"` +} + +type VMMetadata struct { + MeteringImageId string `json:"metering.image_id"` + MeteringImagetype string `json:"metering.imagetype"` + MeteringOrderId string `json:"metering.order_id"` + MeteringResourcespeccode string `json:"metering.resourcespeccode"` + ImageName string `json:"image_name"` + OSBit string `json:"os_bit"` + VpcId string `json:"vpc_id"` + MeteringResourcetype string `json:"metering.resourcetype"` + CascadedInstanceExtrainfo string `json:"cascaded.instance_extrainfo"` + OSType string `json:"os_type"` + ChargingMode string `json:"charging_mode"` +} + +type OSExtendedVolumesVolumesAttached struct { + Device string `json:"device"` + BootIndex string `json:"bootIndex"` + Id string `json:"id"` + DeleteOnTermination string `json:"delete_on_termination"` +} + +type OSSchedulerHints struct { +} + +type SecurityGroup struct { + Name string `json:"name"` +} + +type SysTag struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0094148849.html +// https://support.huaweicloud.com/api-bpconsole/zh-cn_topic_0100166287.html v1.1 支持创建包年/包月的弹性云服务器 +type SInstance struct { + multicloud.SInstanceBase + multicloud.HuaweiTags + + host *SHost + + osInfo *imagetools.ImageInfo + + Id string `json:"id"` + Name string `json:"name"` + Addresses map[string][]IpAddress `json:"addresses"` + Flavor Flavor `json:"flavor"` + AccessIPv4 string `json:"accessIPv4"` + AccessIPv6 string `json:"accessIPv6"` + Status string `json:"status"` + Progress string `json:"progress"` + HostId string `json:"hostId"` + Image Image `json:"image"` + Updated string `json:"updated"` + Created time.Time `json:"created"` + Metadata VMMetadata `json:"metadata"` + Description string `json:"description"` + Locked bool `json:"locked"` + ConfigDrive string `json:"config_drive"` + TenantId string `json:"tenant_id"` + UserId string `json:"user_id"` + KeyName string `json:"key_name"` + + OSExtendedVolumesVolumesAttached []OSExtendedVolumesVolumesAttached `json:"os-extended-volumes:volumes_attached"` + OSEXTSTSTaskState string `json:"OS-EXT-STS:task_state"` + OSEXTSTSPowerState int64 `json:"OS-EXT-STS:power_state"` + OSEXTSTSVMState string `json:"OS-EXT-STS:vm_state"` + OSEXTSRVATTRHost string `json:"OS-EXT-SRV-ATTR:host"` + OSEXTSRVATTRInstanceName string `json:"OS-EXT-SRV-ATTR:instance_name"` + OSEXTSRVATTRHypervisorHostname string `json:"OS-EXT-SRV-ATTR:hypervisor_hostname"` + OSDCFDiskConfig string `json:"OS-DCF:diskConfig"` + OSEXTAZAvailabilityZone string `json:"OS-EXT-AZ:availability_zone"` + OSSchedulerHints OSSchedulerHints `json:"os:scheduler_hints"` + OSEXTSRVATTRRootDeviceName string `json:"OS-EXT-SRV-ATTR:root_device_name"` + OSEXTSRVATTRRamdiskId string `json:"OS-EXT-SRV-ATTR:ramdisk_id"` + EnterpriseProjectId string `json:"enterprise_project_id"` + OSEXTSRVATTRUserData string `json:"OS-EXT-SRV-ATTR:user_data"` + OSSRVUSGLaunchedAt time.Time `json:"OS-SRV-USG:launched_at"` + OSEXTSRVATTRKernelId string `json:"OS-EXT-SRV-ATTR:kernel_id"` + OSEXTSRVATTRLaunchIndex int64 `json:"OS-EXT-SRV-ATTR:launch_index"` + HostStatus string `json:"host_status"` + OSEXTSRVATTRReservationId string `json:"OS-EXT-SRV-ATTR:reservation_id"` + OSEXTSRVATTRHostname string `json:"OS-EXT-SRV-ATTR:hostname"` + OSSRVUSGTerminatedAt time.Time `json:"OS-SRV-USG:terminated_at"` + SysTags []SysTag `json:"sys_tags"` + SecurityGroups []SecurityGroup `json:"security_groups"` +} + +func compareSet(currentSet []string, newSet []string) (add []string, remove []string, keep []string) { + sort.Strings(currentSet) + sort.Strings(newSet) + + i, j := 0, 0 + for i < len(currentSet) || j < len(newSet) { + if i < len(currentSet) && j < len(newSet) { + if currentSet[i] == newSet[j] { + keep = append(keep, currentSet[i]) + i += 1 + j += 1 + } else if currentSet[i] < newSet[j] { + remove = append(remove, currentSet[i]) + i += 1 + } else { + add = append(add, newSet[j]) + j += 1 + } + } else if i >= len(currentSet) { + add = append(add, newSet[j]) + j += 1 + } else if j >= len(newSet) { + remove = append(remove, currentSet[i]) + i += 1 + } + } + + return add, remove, keep +} + +// 启动盘 != 系统盘(必须是启动盘且挂载在root device上) +func isBootDisk(server *SInstance, disk *SDisk) bool { + if disk.GetDiskType() != api.DISK_TYPE_SYS { + return false + } + + for _, attachment := range disk.Attachments { + if attachment.ServerId == server.GetId() && attachment.Device == server.OSEXTSRVATTRRootDeviceName { + return true + } + } + + return false +} + +func (self *SInstance) GetId() string { + return self.Id +} + +func (self *SInstance) GetHostname() string { + return self.OSEXTSRVATTRHostname +} + +func (self *SInstance) GetName() string { + return self.Name +} + +func (self *SInstance) GetGlobalId() string { + return self.Id +} + +func (self *SInstance) GetStatus() string { + switch self.Status { + case "ACTIVE": + return api.VM_RUNNING + case "MIGRATING", "REBUILD", "BUILD", "RESIZE", "VERIFY_RESIZE": // todo: pending ? + return api.VM_STARTING + case "REBOOT", "HARD_REBOOT": + return api.VM_STOPPING + case "SHUTOFF": + return api.VM_READY + default: + return api.VM_UNKNOWN + } +} + +func (self *SInstance) Refresh() error { + ret, err := self.host.zone.region.GetInstance(self.GetId()) + if err != nil { + return err + } + return jsonutils.Update(self, ret) +} + +func (self *SInstance) GetInstanceType() string { + return self.Flavor.Id +} + +func (self *SInstance) GetSecurityGroupIds() ([]string, error) { + secgroups, err := self.host.zone.region.GetInstanceSecrityGroups(self.Id) + if err != nil { + return nil, err + } + ret := []string{} + for i := range secgroups { + ret = append(ret, secgroups[i].Id) + } + return ret, nil +} + +// https://support.huaweicloud.com/api-ecs/ecs_02_1002.html +// key 相同时value不会替换 +func (self *SRegion) CreateServerTags(instanceId string, tags map[string]string) error { + params := map[string]interface{}{ + "action": "create", + } + + tagsObj := []map[string]string{} + for k, v := range tags { + tagsObj = append(tagsObj, map[string]string{"key": k, "value": v}) + } + params["tags"] = tagsObj + + res := fmt.Sprintf("cloudservers/%s", instanceId) + return self.perform("ecs", "v1", res, "tags/action", params, nil) +} + +// https://support.huaweicloud.com/api-ecs/ecs_02_1003.html +func (self *SRegion) DeleteServerTags(instanceId string, tagsKey []string) error { + params := map[string]interface{}{ + "action": "delete", + } + tagsObj := []map[string]string{} + for _, k := range tagsKey { + tagsObj = append(tagsObj, map[string]string{"key": k}) + } + params["tags"] = tagsObj + res := fmt.Sprintf("cloudservers/%s", instanceId) + return self.perform("ecs", "v1", res, "tags/action", params, nil) +} + +func (self *SInstance) SetTags(tags map[string]string, replace bool) error { + existedTags, err := self.GetTags() + if err != nil { + return errors.Wrap(err, "self.GetTags()") + } + deleteTagsKey := []string{} + for k := range existedTags { + if replace { + deleteTagsKey = append(deleteTagsKey, k) + } else { + if _, ok := tags[k]; ok { + deleteTagsKey = append(deleteTagsKey, k) + } + } + } + if len(deleteTagsKey) > 0 { + err := self.host.zone.region.DeleteServerTags(self.GetId(), deleteTagsKey) + if err != nil { + return errors.Wrapf(err, "self.host.zone.region.DeleteServerTags(%s,%s)", self.GetId(), deleteTagsKey) + } + } + if len(tags) > 0 { + err := self.host.zone.region.CreateServerTags(self.GetId(), tags) + if err != nil { + return errors.Wrapf(err, "self.host.zone.region.CreateServerTags(%s,%s)", self.GetId(), jsonutils.Marshal(tags).String()) + } + } + return nil +} + +func (self *SInstance) GetBillingType() string { + // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0094148849.html + // charging_mode “0”：按需计费 “1”：按包年包月计费 + if self.Metadata.ChargingMode == "1" { + return billing_api.BILLING_TYPE_PREPAID + } + return billing_api.BILLING_TYPE_POSTPAID +} + +func (self *SInstance) GetCreatedAt() time.Time { + return self.Created +} + +// charging_mode “0”：按需计费 “1”：按包年包月计费 +func (self *SInstance) GetExpiredAt() time.Time { + return time.Time{} +} + +func (self *SInstance) GetIHost() cloudprovider.ICloudHost { + return self.host +} + +func (self *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { + attached := self.OSExtendedVolumesVolumesAttached + disks := make([]SDisk, 0) + for _, vol := range attached { + disk, err := self.host.zone.region.GetDisk(vol.Id) + if err != nil { + return nil, err + } + + disks = append(disks, *disk) + } + + ret := []cloudprovider.ICloudDisk{} + for i := 0; i < len(disks); i += 1 { + disks[i].region = self.host.zone.region + if isBootDisk(self, &disks[i]) { + ret = append([]cloudprovider.ICloudDisk{&disks[i]}, ret...) + } else { + ret = append(ret, &disks[i]) + } + } + return ret, nil +} + +func (self *SInstance) GetINics() ([]cloudprovider.ICloudNic, error) { + nics := make([]cloudprovider.ICloudNic, 0) + for _, ipAddresses := range self.Addresses { + for _, ipAddress := range ipAddresses { + if ipAddress.OSEXTIPSType == "fixed" { + nic := SInstanceNic{ + instance: self, + ipAddr: ipAddress.Addr, + macAddr: ipAddress.OSEXTIPSMACMACAddr, + } + nics = append(nics, &nic) + } + } + } + return nics, nil +} + +func (self *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { + ips := make([]string, 0) + for _, addresses := range self.Addresses { + for _, address := range addresses { + if address.OSEXTIPSType != "fixed" && !strings.HasPrefix(address.Addr, "100.") { + ips = append(ips, address.Addr) + } + } + } + + if len(ips) == 0 { + return nil, nil + } + + eips, err := self.host.zone.region.GetEips("", ips) + if err != nil { + return nil, err + } + if len(eips) > 0 { + eips[0].region = self.host.zone.region + return &eips[0], nil + } + return nil, nil +} + +func (self *SInstance) GetVcpuCount() int { + cpu, _ := strconv.Atoi(self.Flavor.Vcpus) + return cpu +} + +func (self *SInstance) GetVmemSizeMB() int { + mem, _ := strconv.Atoi(self.Flavor.RAM) + return int(mem) +} + +func (self *SInstance) GetBootOrder() string { + return "dcn" +} + +func (self *SInstance) GetVga() string { + return "std" +} + +func (self *SInstance) GetVdi() string { + return "vnc" +} + +func (self *SInstance) GetOSArch() string { + if len(self.Image.Id) > 0 { + image, err := self.host.zone.region.GetImage(self.Image.Id) + if err == nil { + return image.GetOsArch() + } + + log.Debugf("GetOSArch.GetImage %s: %s", self.Image.Id, err) + } + + t := self.GetInstanceType() + if len(t) > 0 { + if strings.HasPrefix(t, "k") { + return apis.OS_ARCH_AARCH64 + } + } + + return apis.OS_ARCH_X86 +} + +func (self *SInstance) GetOsType() cloudprovider.TOsType { + return cloudprovider.TOsType(osprofile.NormalizeOSType(self.Metadata.OSType)) +} + +func (self *SInstance) GetOSName() string { + return self.Metadata.ImageName +} + +func (i *SInstance) getNormalizedOsInfo() *imagetools.ImageInfo { + if i.osInfo == nil { + osInfo := imagetools.NormalizeImageInfo(i.Metadata.ImageName, "", i.Metadata.OSType, "", "") + i.osInfo = &osInfo + } + return i.osInfo +} + +func (self *SInstance) GetBios() cloudprovider.TBiosType { + return cloudprovider.BIOS +} + +func (i *SInstance) GetOsDist() string { + return i.getNormalizedOsInfo().OsDistro +} + +func (i *SInstance) GetOsVersion() string { + return i.getNormalizedOsInfo().OsVersion +} + +func (i *SInstance) GetOsLang() string { + return i.getNormalizedOsInfo().OsLang +} + +func (self *SInstance) GetFullOsName() string { + return self.Metadata.ImageName +} + +func (self *SInstance) GetMachine() string { + return "pc" +} + +func (self *SInstance) GetOsArch() string { + if flavor, err := self.host.zone.region.GetICloudSku(self.Flavor.Id); err == nil { + return flavor.GetCpuArch() + } else { + log.Debugf("GetOSArch.GetICloudSku %s: %s", self.Flavor.Id, err) + } + + t := self.GetInstanceType() + if len(t) > 0 { + if strings.HasPrefix(t, "k") { + return apis.OS_ARCH_AARCH64 + } + } + + return apis.OS_ARCH_X86 +} + +func (self *SInstance) AssignSecurityGroup(secgroupId string) error { + return self.SetSecurityGroups([]string{secgroupId}) +} + +func (self *SInstance) SetSecurityGroups(secgroupIds []string) error { + currentSecgroups, err := self.GetSecurityGroupIds() + if err != nil { + return err + } + + add, remove, _ := compareSet(currentSecgroups, secgroupIds) + err = self.host.zone.region.AssignSecurityGroups(add, self.GetId()) + if err != nil { + return err + } + return self.host.zone.region.UnassignSecurityGroups(remove, self.GetId()) +} + +func (self *SInstance) GetHypervisor() string { + return api.HYPERVISOR_HCS +} + +func (self *SInstance) StartVM(ctx context.Context) error { + if self.Status == InstanceStatusRunning { + return nil + } + + timeout := 300 * time.Second + interval := 15 * time.Second + + startTime := time.Now() + for time.Now().Sub(startTime) < timeout { + err := self.Refresh() + if err != nil { + return err + } + + if self.GetStatus() == api.VM_RUNNING { + return nil + } else if self.GetStatus() == api.VM_READY { + err := self.host.zone.region.StartVM(self.GetId()) + if err != nil { + return err + } + } + time.Sleep(interval) + } + return cloudprovider.ErrTimeout +} + +func (self *SInstance) StopVM(ctx context.Context, opts *cloudprovider.ServerStopOptions) error { + if self.Status == InstanceStatusStopped { + return nil + } + + if self.Status == InstanceStatusTerminated { + log.Debugf("Instance already terminated.") + return nil + } + + err := self.host.zone.region.StopVM(self.GetId(), opts.IsForce) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, api.VM_READY, 10*time.Second, 300*time.Second) // 5mintues +} + +func (self *SInstance) DeleteVM(ctx context.Context) error { + return self.host.zone.region.DeleteVM(self.GetId()) +} + +func (self *SInstance) UpdateVM(ctx context.Context, name string) error { + return self.host.zone.region.UpdateVM(self.GetId(), name) +} + +// https://support.huaweicloud.com/usermanual-ecs/zh-cn_topic_0032380449.html +// 创建云服务器过程中注入用户数据。支持注入文本、文本文件或gzip文件。 +// 注入内容，需要进行base64格式编码。注入内容（编码之前的内容）最大长度32KB。 +// 对于Linux弹性云服务器，adminPass参数传入时，user_data参数不生效。 +func (self *SInstance) UpdateUserData(userData string) error { + return cloudprovider.ErrNotSupported +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876349.html 使用原镜像重装 +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876971.html 更换系统盘操作系统 +// 不支持调整系统盘大小 +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { + publicKeyName := "" + var err error + if len(desc.PublicKey) > 0 { + publicKeyName, err = self.host.zone.region.syncKeypair(desc.PublicKey) + if err != nil { + return "", err + } + } + + image, err := self.host.zone.region.GetImage(desc.ImageId) + if err != nil { + return "", errors.Wrap(err, "SInstance.RebuildRoot.GetImage") + } + + // Password存在的情况下，windows 系统直接使用密码 + if strings.ToLower(image.Platform) == strings.ToLower(osprofile.OS_TYPE_WINDOWS) && len(desc.Password) > 0 { + publicKeyName = "" + } + + userData, err := updateUserData(self.OSEXTSRVATTRUserData, image.OSVersion, desc.Account, desc.Password, desc.PublicKey) + if err != nil { + return "", errors.Wrap(err, "SInstance.RebuildRoot.updateUserData") + } + + if self.Metadata.MeteringImageId == desc.ImageId { + err = self.host.zone.region.RebuildRoot(ctx, self.UserId, self.GetId(), desc.Password, publicKeyName, userData) + if err != nil { + return "", err + } + } else { + err = self.host.zone.region.ChangeRoot(ctx, self.UserId, self.GetId(), desc.ImageId, desc.Password, publicKeyName, userData) + if err != nil { + return "", err + } + } + + idisks, err := self.GetIDisks() + if err != nil { + return "", err + } + + if len(idisks) == 0 { + return "", fmt.Errorf("server %s has no volume attached.", self.GetId()) + } + + return idisks[0].GetId(), nil +} + +func (self *SInstance) DeployVM(ctx context.Context, name string, username string, password string, publicKey string, deleteKeypair bool, description string) error { + return self.host.zone.region.DeployVM(self.GetId(), name, password, publicKey, deleteKeypair, description) +} + +func (self *SInstance) ChangeConfig(ctx context.Context, config *cloudprovider.SManagedVMChangeConfig) error { + instanceTypes := []string{} + if len(config.InstanceType) > 0 { + instanceTypes = []string{config.InstanceType} + } else { + flavors, err := self.host.zone.region.GetMatchInstanceTypes(config.Cpu, config.MemoryMB, self.OSEXTAZAvailabilityZone) + if err != nil { + return errors.Wrapf(err, "GetMatchInstanceTypes") + } + for _, flavor := range flavors { + instanceTypes = append(instanceTypes, flavor.Id) + } + } + var err error + for _, instanceType := range instanceTypes { + err = self.host.zone.region.ChangeVMConfig(self.GetId(), instanceType) + if err != nil { + log.Warningf("ChangeVMConfig %s for %s error: %v", self.GetId(), instanceType, err) + } else { + return cloudprovider.WaitStatusWithDelay(self, api.VM_READY, 15*time.Second, 15*time.Second, 180*time.Second) + } + } + if err != nil { + return errors.Wrapf(err, "ChangeVMConfig") + } + return fmt.Errorf("Failed to change vm config, specification not supported") +} + +// todo:// 返回jsonobject感觉很诡异。不能直接知道内部细节 +func (self *SInstance) GetVNCInfo(input *cloudprovider.ServerVncInput) (*cloudprovider.ServerVncOutput, error) { + return self.host.zone.region.GetInstanceVNCUrl(self.GetId()) +} + +func (self *SInstance) NextDeviceName() (string, error) { + prefix := "s" + if strings.Contains(self.OSEXTSRVATTRRootDeviceName, "/vd") { + prefix = "v" + } + + currents := []string{} + for _, item := range self.OSExtendedVolumesVolumesAttached { + currents = append(currents, strings.ToLower(item.Device)) + } + + for i := 0; i < 25; i++ { + device := fmt.Sprintf("/dev/%sd%s", prefix, string([]byte{byte(98 + i)})) + if ok, _ := utils.InStringArray(device, currents); !ok { + return device, nil + } + } + + return "", fmt.Errorf("disk devicename out of index, current deivces: %s", currents) +} + +func (self *SInstance) AttachDisk(ctx context.Context, diskId string) error { + device, err := self.NextDeviceName() + if err != nil { + return errors.Wrap(err, "Instance.AttachDisk.NextDeviceName") + } + + err = self.host.zone.region.AttachDisk(self.GetId(), diskId, device) + if err != nil { + return errors.Wrap(err, "Instance.AttachDisk.AttachDisk") + } + + return cloudprovider.Wait(5*time.Second, 60*time.Second, func() (bool, error) { + disk, err := self.host.zone.region.GetDisk(diskId) + if err != nil { + log.Debugf("Instance.AttachDisk.GetDisk %s", err) + return false, nil + } + + if disk.Status == "in-use" { + return true, nil + } + + return false, nil + }) +} + +func (self *SInstance) DetachDisk(ctx context.Context, diskId string) error { + err := self.host.zone.region.DetachDisk(self.GetId(), diskId) + if err != nil { + return errors.Wrap(err, "Instance.DetachDisk") + } + + return cloudprovider.Wait(5*time.Second, 60*time.Second, func() (bool, error) { + disk, err := self.host.zone.region.GetDisk(diskId) + if err != nil { + log.Debugf("Instance.DetachDisk.GetDisk %s", err) + return false, nil + } + + if disk.Status == "available" { + return true, nil + } + + return false, nil + }) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0094148850.html +func (self *SRegion) GetInstances(ip string) ([]SInstance, error) { + params := url.Values{} + if len(ip) > 0 { + params.Set("ip", ip) + } + params.Set("offset", "1") + ret := []SInstance{} + return ret, self.list("ecs", "v1", "cloudservers/detail", params, &ret) +} + +func (self *SRegion) GetInstance(id string) (*SInstance, error) { + ret := &SInstance{} + res := fmt.Sprintf("cloudservers/%s", id) + return ret, self.get("ecs", "v1", res, ret) +} +func (self *SRegion) CreateInstance(name string, imageId string, instanceType string, subnetId string, + securityGroupId string, vpcId string, zoneId string, desc string, sysDisk cloudprovider.SDiskInfo, dataDisks []cloudprovider.SDiskInfo, ipAddr string, + keypair string, passwd string, userData string, bc *billing.SBillingCycle, projectId string, tags map[string]string) (*SInstance, error) { + diskParams := []map[string]interface{}{} + for _, disk := range dataDisks { + diskParams = append(diskParams, map[string]interface{}{ + "volumetype": disk.StorageType, + "size": disk.SizeGB, + }) + } + params := map[string]interface{}{ + "availability_zone": zoneId, + "name": name, + "flavorRef": instanceType, + "imageRef": imageId, + "description": desc, + "count": 1, + "tenancy": "0", + "region_id": self.Id, + "operate_type": "apply", + "service_type": "ecs", + "tenantId": self.client.projectId, + "project_id": self.client.projectId, + "nics": []map[string]interface{}{ + { + "subnet_id": subnetId, + "ip_address": ipAddr, + "binding:profile": map[string]interface{}{ + "disable_security_groups": false, + }, + }, + }, + "security_groups": []map[string]interface{}{ + { + "id": securityGroupId, + }, + }, + "vpcid": vpcId, + "metadata": map[string]string{}, + "root_volume": map[string]interface{}{ + "volumetype": sysDisk.StorageType, + "size": sysDisk.SizeGB, + }, + "data_volumes": diskParams, + "extendparam": map[string]interface{}{ + "enterprise_project_id": projectId, + "regionID": self.Id, + }, + } + + if len(keypair) > 0 { + params["key_name"] = keypair + } else { + params["adminPass"] = passwd + } + + if len(userData) > 0 { + params["user_data"] = userData + } + tagParams := []map[string]interface{}{} + for k, v := range tags { + tagParams = append(tagParams, map[string]interface{}{ + "key": k, + "value": v, + }) + } + params["server_tags"] = tagParams + body := map[string]interface{}{ + "server": params, + } + job := &SJob{} + err := self.client.create("ecs", "v1.1", self.Id, "cloudservers", body, job) + if err != nil { + return nil, err + } + for _, sub := range job.Entities.SubJobs { + if len(sub.Entities.ServerId) > 0 { + return self.GetInstance(sub.Entities.ServerId) + } + } + return nil, fmt.Errorf("no server id returned with job %s", jsonutils.Marshal(job)) +} + +func (self *SRegion) AssignSecurityGroups(secgroupIds []string, instanceId string) error { + return cloudprovider.ErrNotImplemented +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067161717.html +func (self *SRegion) UnassignSecurityGroups(secgroupIds []string, instanceId string) error { + return cloudprovider.ErrNotImplemented +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212207.html +func (self *SRegion) StartVM(id string) error { + params := map[string]interface{}{ + "os-start": map[string]string{}, + } + res := fmt.Sprintf("servers/%s", id) + return self.ecsPerform(res, "action", params, nil) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212651.html +func (self *SRegion) StopVM(id string, isForce bool) error { + params := map[string]interface{}{ + "os-stop": map[string]string{}, + } + res := fmt.Sprintf("servers/%s", id) + return self.ecsPerform(res, "action", params, nil) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212679.html +// 只删除主机。弹性IP和数据盘需要单独删除 +func (self *SRegion) DeleteVM(id string) error { + params := map[string]interface{}{ + "delete_publicip": false, + "delete_volume": false, + "servers": []struct { + Id string + }{ + {Id: id}, + }, + } + return self.perform("ecs", "v1", "cloudservers", "delete", params, nil) +} + +func (self *SRegion) UpdateVM(instanceId, name string) error { + params := map[string]interface{}{ + "server": map[string]string{ + "name": name, + }, + } + return self.update("ecs", "v2.1", "servers/"+instanceId, params) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876349.html +// 返回job id +func (self *SRegion) RebuildRoot(ctx context.Context, userId, instanceId, passwd, publicKeyName, userData string) error { + reInstall := map[string]interface{}{} + if len(publicKeyName) > 0 { + reInstall["keyname"] = publicKeyName + } else if len(passwd) > 0 { + reInstall["adminpass"] = passwd + } else { + return fmt.Errorf("both password and publicKey are empty.") + } + if len(userId) > 0 { + reInstall["userid"] = userId + } + metadata := map[string]interface{}{} + if len(userData) > 0 { + metadata["user_data"] = userData + } + reInstall["metadata"] = metadata + + params := map[string]interface{}{ + "os-reinstall": reInstall, + } + return self.ecsPerform("cloudservers/"+instanceId, "reinstallos", params, nil) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876971.html +// 返回job id +func (self *SRegion) ChangeRoot(ctx context.Context, userId, instanceId, imageId, passwd, publicKeyName, userData string) error { + osChange := map[string]interface{}{} + if len(publicKeyName) > 0 { + osChange["keyname"] = publicKeyName + } else if len(passwd) > 0 { + osChange["adminpass"] = passwd + } else { + return fmt.Errorf("both password and publicKey are empty.") + } + if len(userId) > 0 { + osChange["userid"] = userId + } + metadata := map[string]interface{}{} + if len(userData) > 0 { + metadata["user_data"] = userData + } + osChange["metadata"] = metadata + osChange["imageid"] = imageId + params := map[string]interface{}{ + "os-change": osChange, + } + return self.ecsPerform("cloudservers/"+instanceId, "changeos", params, nil) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212692.html +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0110109377.html +// 一键式重置密码 需要安装安装一键式重置密码插件 https://support.huaweicloud.com/usermanual-ecs/zh-cn_topic_0068095385.html +// 目前不支持直接重置密钥 +func (self *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, deleteKeypair bool, description string) error { + if len(name) > 0 { + err := self.UpdateVM(instanceId, name) + if err != nil { + return err + } + } + + if len(password) > 0 { + params := map[string]interface{}{ + "reset-password": map[string]interface{}{ + "new_password": password, + }, + } + return self.perform("ecs", "v1", "cloudservers/"+instanceId, "os-reset-password", params, nil) + } + return nil +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212653.html +func (self *SRegion) ChangeVMConfig(instanceId string, instanceType string) error { + params := map[string]interface{}{ + "resize": map[string]interface{}{ + "flavorRef": instanceType, + }, + } + return self.perform("ecs", "v1.1", "cloudservers/"+instanceId, "resize", params, nil) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0142763126.html 微版本2.6及以上? +// https://support.huaweicloud.com/api-ecs/ecs_02_0208.html +func (self *SRegion) GetInstanceVNCUrl(instanceId string) (*cloudprovider.ServerVncOutput, error) { + params := map[string]interface{}{ + "remote_console": map[string]interface{}{ + "type": "novnc", + "protocol": "vnc", + }, + } + ret := &cloudprovider.ServerVncOutput{} + err := self.perform("ecs", "v1", "cloudservers/"+instanceId, "remote_console", params, ret) + if err != nil { + return nil, err + } + ret.Hypervisor = api.HYPERVISOR_HUAWEI + ret.Protocol = "huawei" + return ret, nil +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0065817702.html +func (self *SRegion) GetInstanceSecrityGroups(instanceId string) ([]SSecurityGroup, error) { + ret := []SSecurityGroup{} + return ret, self.get("ecs", "v2.1", "servers/"+instanceId+"/os-security-groups", &ret) +} + +func (self *SInstance) GetProjectId() string { + return self.EnterpriseProjectId +} + +func (self *SInstance) GetError() error { + return nil +} + +func updateUserData(userData, osVersion, username, password, publicKey string) (string, error) { + winOS := strings.ToLower(osprofile.OS_TYPE_WINDOWS) + osVersion = strings.ToLower(osVersion) + config := &cloudinit.SCloudConfig{} + if len(userData) > 0 { + if strings.Contains(osVersion, winOS) { + if _config, err := cloudinit.ParseUserDataBase64(userData); err == nil { + config = _config + } else { + log.Debugf("updateWindowsUserData invalid userdata %s", userData) + } + } else { + if _config, err := cloudinit.ParseUserDataBase64(userData); err == nil { + config = _config + } else { + return "", fmt.Errorf("updateLinuxUserData invalid userdata %s", userData) + } + } + } + + user := cloudinit.NewUser(username) + config.RemoveUser(user) + config.DisableRoot = 0 + if len(password) > 0 { + config.SshPwauth = cloudinit.SSH_PASSWORD_AUTH_ON + user.Password(password) + config.MergeUser(user) + } + + if len(publicKey) > 0 { + user.SshKey(publicKey) + config.MergeUser(user) + } + + if strings.Contains(osVersion, winOS) { + userData, err := updateWindowsUserData(config.UserDataPowerShell(), osVersion, username, password) + if err != nil { + return "", errors.Wrap(err, "updateUserData.updateWindowsUserData") + } + return userData, nil + } else { + return config.UserDataBase64(), nil + } +} + +func updateWindowsUserData(userData string, osVersion string, username, password string) (string, error) { + // Windows Server 2003, Windows Vista, Windows Server 2008, Windows Server 2003 R2, Windows Server 2000, Windows Server 2012, Windows Server 2003 with SP1, Windows 8 + oldVersions := []string{"2000", "2003", "2008", "2012", "Vista"} + isOldVersion := false + for i := range oldVersions { + if strings.Contains(osVersion, oldVersions[i]) { + isOldVersion = true + } + } + + shells := "" + if isOldVersion { + shells += fmt.Sprintf("rem cmd\n") + if username == "Administrator" { + shells += fmt.Sprintf("net user %s %s\n", username, password) + } else { + shells += fmt.Sprintf("net user %s %s /add\n", username, password) + shells += fmt.Sprintf("net localgroup administrators %s /add\n", username) + } + + shells += fmt.Sprintf("net user %s /active:yes", username) + } else { + if !strings.HasPrefix(userData, "#ps1") { + shells = fmt.Sprintf("#ps1\n%s", userData) + } + } + + return base64.StdEncoding.EncodeToString([]byte(shells)), nil +} + +func (self *SRegion) SaveImage(instanceId string, opts *cloudprovider.SaveImageOptions) (*SImage, error) { + params := map[string]interface{}{ + "name": opts.Name, + "instance_id": instanceId, + } + if len(opts.Notes) > 0 { + params["description"] = func() string { + opts.Notes = strings.ReplaceAll(opts.Notes, "<", "") + opts.Notes = strings.ReplaceAll(opts.Notes, ">", "") + opts.Notes = strings.ReplaceAll(opts.Notes, "\n", "") + if len(opts.Notes) > 1024 { + opts.Notes = opts.Notes[:1024] + } + return opts.Notes + }() + } + ret := &SImage{cache: &SStoragecache{region: self}} + return ret, self.perform("ims", "v2", "cloudimages", "action", params, ret) +} + +func (self *SInstance) SaveImage(opts *cloudprovider.SaveImageOptions) (cloudprovider.ICloudImage, error) { + image, err := self.host.zone.region.SaveImage(self.Id, opts) + if err != nil { + return nil, errors.Wrapf(err, "SaveImage") + } + return image, nil +} diff --git a/pkg/multicloud/hcs/instancenic.go b/pkg/multicloud/hcs/instancenic.go new file mode 100644 index 0000000000..76a09558c7 --- /dev/null +++ b/pkg/multicloud/hcs/instancenic.go @@ -0,0 +1,86 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +// =========================================== +type Interface struct { + PortState string `json:"port_state"` + FixedIPS []FixedIP `json:"fixed_ips"` + NetId string `json:"net_id"` // 网络Id. 与 SNetwork里的Id对应。统一使用这个Id + PortId string `json:"port_id"` + MACAddr string `json:"mac_addr"` +} + +type FixedIP struct { + SubnetId string `json:"subnet_id"` // 子网Id, 与SNetwork中的 neutron_subnet_id对应. 注意!!! 并不是SNetwork Id。 + IPAddress string `json:"ip_address"` +} + +type SInstanceNic struct { + instance *SInstance + ipAddr string + macAddr string + + cloudprovider.DummyICloudNic +} + +func (self *SInstanceNic) GetId() string { + return "" +} + +func (self *SInstanceNic) GetIP() string { + return self.ipAddr +} + +func (self *SInstanceNic) GetMAC() string { + return self.macAddr +} + +func (self *SInstanceNic) GetDriver() string { + return "virtio" +} + +func (self *SInstanceNic) InClassicNetwork() bool { + return false +} + +func (self *SInstanceNic) GetINetworkId() string { + instanceId := self.instance.GetId() + subnets, _ := self.instance.host.zone.region.GetInstanceInterfaces(instanceId) + for i := range subnets { + return subnets[i].NetId + } + return "" +} + +type SInstanceInterface struct { + NetId string + PortId string + MacAddr string + PortState string + FixedIps []struct { + SubnetId string + IpAddress string + } +} + +func (self *SRegion) GetInstanceInterfaces(instanceId string) ([]SInstanceInterface, error) { + ret := []SInstanceInterface{} + return ret, self.get("ecs", "v2.1", "servers/"+instanceId+"/os-interface", &ret) +} diff --git a/pkg/multicloud/hcs/instancetype.go b/pkg/multicloud/hcs/instancetype.go new file mode 100644 index 0000000000..3de89094d8 --- /dev/null +++ b/pkg/multicloud/hcs/instancetype.go @@ -0,0 +1,223 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212656.html +type SInstanceType struct { + multicloud.SResourceBase + multicloud.HcsTags + + Id string `json:"id"` + Name string `json:"name"` + Vcpus int `json:"vcpus"` + RamMB int `json:"ram"` // 内存大小 + OSExtraSpecs OSExtraSpecs `json:"os_extra_specs"` // 扩展规格 +} + +type OSExtraSpecs struct { + EcsPerformancetype string `json:"ecs:performancetype"` + EcsGeneration string `json:"ecs:generation"` + EcsInstanceArchitecture string `json:"ecs:instance_architecture"` +} + +func (self *SInstanceType) GetId() string { + return self.Id +} + +func (self *SInstanceType) GetName() string { + return self.Id +} + +func (self *SInstanceType) GetGlobalId() string { + return self.Id +} + +func (self *SInstanceType) GetInstanceTypeFamily() string { + if len(self.OSExtraSpecs.EcsGeneration) > 0 { + return self.OSExtraSpecs.EcsGeneration + } + return strings.Split(self.Name, ".")[0] +} + +func (self *SInstanceType) GetStatus() string { + return "" +} + +func (self *SInstanceType) GetInstanceTypeCategory() string { + return self.OSExtraSpecs.EcsPerformancetype +} + +func (self *SInstanceType) GetPrepaidStatus() string { + return api.SkuStatusSoldout +} + +func (self *SInstanceType) GetPostpaidStatus() string { + return api.SkuStatusAvailable +} + +func (self *SInstanceType) GetCpuArch() string { + if len(self.OSExtraSpecs.EcsInstanceArchitecture) > 0 { + if strings.HasPrefix(self.OSExtraSpecs.EcsInstanceArchitecture, "arm") { + return apis.OS_ARCH_AARCH64 + } + return apis.OS_ARCH_X86 + } + return "" +} + +func (self *SInstanceType) GetCpuCoreCount() int { + return self.Vcpus +} + +func (self *SInstanceType) GetMemorySizeMB() int { + return self.RamMB +} + +func (self *SInstanceType) GetOsName() string { + return "" +} + +func (self *SInstanceType) GetSysDiskResizable() bool { + return false +} + +func (self *SInstanceType) GetSysDiskType() string { + return "" +} + +func (self *SInstanceType) GetSysDiskMinSizeGB() int { + return 0 +} + +func (self *SInstanceType) GetSysDiskMaxSizeGB() int { + return 0 +} + +func (self *SInstanceType) GetAttachedDiskType() string { + return "" +} + +func (self *SInstanceType) GetAttachedDiskSizeGB() int { + return 0 +} + +func (self *SInstanceType) GetAttachedDiskCount() int { + return 0 +} + +func (self *SInstanceType) GetDataDiskTypes() string { + return "" +} + +func (self *SInstanceType) GetDataDiskMaxCount() int { + return 0 +} + +func (self *SInstanceType) GetNicType() string { + return "" +} + +func (self *SInstanceType) GetNicMaxCount() int { + return 0 +} + +func (self *SInstanceType) GetGpuAttachable() bool { + return self.OSExtraSpecs.EcsPerformancetype == "gpu" +} + +func (self *SInstanceType) GetGpuSpec() string { + if self.OSExtraSpecs.EcsPerformancetype == "gpu" { + return self.OSExtraSpecs.EcsGeneration + } + + return "" +} + +func (self *SInstanceType) GetGpuCount() int { + if self.OSExtraSpecs.EcsPerformancetype == "gpu" { + return 1 + } + + return 0 +} + +func (self *SInstanceType) GetGpuMaxCount() int { + if self.OSExtraSpecs.EcsPerformancetype == "gpu" { + return 1 + } + + return 0 +} + +func (self *SInstanceType) Delete() error { + return nil +} + +func (self *SRegion) GetchInstanceTypes(zoneId string) ([]SInstanceType, error) { + query := url.Values{} + if len(zoneId) > 0 { + query.Set("availability_zone", zoneId) + } + ret := []SInstanceType{} + return ret, self.list("ecs", "v1", "cloudservers/flavors", query, &ret) +} + +func (self *SRegion) GetchInstanceType(id string) (*SInstanceType, error) { + ret := &SInstanceType{} + res := fmt.Sprintf("flavors/%s", id) + return ret, self.get("ecs", "v2", res, &ret) +} + +func (self *SRegion) GetMatchInstanceTypes(cpu int, memMB int, zoneId string) ([]SInstanceType, error) { + instanceTypes, err := self.GetchInstanceTypes(zoneId) + if err != nil { + return nil, err + } + + ret := make([]SInstanceType, 0) + for _, t := range instanceTypes { + // cpu & mem & disk都匹配才行 + if t.Vcpus == cpu && t.RamMB == memMB { + ret = append(ret, t) + } + } + + return ret, nil +} + +func (self *SRegion) GetISkus() ([]cloudprovider.ICloudSku, error) { + flavors, err := self.GetchInstanceTypes("") + if err != nil { + return nil, errors.Wrap(err, "fetchInstanceTypes") + } + ret := []cloudprovider.ICloudSku{} + for i := range flavors { + ret = append(ret, &flavors[i]) + } + return ret, nil +} diff --git a/pkg/multicloud/hcs/job.go b/pkg/multicloud/hcs/job.go new file mode 100644 index 0000000000..62e158c51c --- /dev/null +++ b/pkg/multicloud/hcs/job.go @@ -0,0 +1,105 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +const ( + TASK_SUCCESS = "SUCCESS" + TASK_FAIL = "FAIL" +) + +type SJob struct { + Status string `json:"status"` + Entities struct { + SubJobsJotal int + ImageId string + ServerId string + SubJobs []SJob + } `json:"entities"` + JobId string `json:"job_id"` + JobType string `json:"job_type"` + BeginTime string `json:"begin_time"` + EndTime string `json:"end_time"` + ErrorCode string `json:"error_code"` + FailReason string `json:"fail_reason"` +} + +func (self *SJob) GetIds() []string { + ret := []string{} + if len(self.Entities.ImageId) > 0 { + ret = append(ret, self.Entities.ImageId) + } + if len(self.Entities.ServerId) > 0 { + ret = append(ret, self.Entities.ServerId) + } + for _, sub := range self.Entities.SubJobs { + if len(sub.Entities.ServerId) > 0 { + ret = append(ret, sub.Entities.ServerId) + } + } + return ret +} + +func (self *SHcsClient) waitJobSuccess(serviceType, regionId string, jobId string, interval time.Duration, timeout time.Duration) (*SJob, error) { + start := time.Now() + var job *SJob + var err error + for time.Now().Sub(start) < timeout { + job, err = self.GetJob(serviceType, regionId, jobId) + if err != nil { + return nil, err + } + log.Infof("wait %s job %s(%s) status: %s", serviceType, job.JobId, job.JobType, job.Status) + if job.Status == TASK_SUCCESS { + return job, nil + } + reason := []string{job.FailReason} + for _, subJob := range job.Entities.SubJobs { + if len(subJob.FailReason) > 0 { + reason = append(reason, subJob.FailReason) + } + } + if job.Status == TASK_FAIL { + return nil, fmt.Errorf("job %s failed reason %s", jobId, strings.Join(reason, ";")) + } + time.Sleep(interval) + } + return nil, errors.Wrapf(cloudprovider.ErrTimeout, "wait job %s", jsonutils.Marshal(job)) +} + +func (self *SHcsClient) GetJob(service, regionId string, jobId string) (*SJob, error) { + resp, err := self._getJob(service, regionId, jobId) + if err != nil { + return nil, err + } + ret := &SJob{} + return ret, resp.Unmarshal(ret) + +} + +func (self *SRegion) GetJob(service string, jobId string) (*SJob, error) { + return self.client.GetJob(service, self.Id, jobId) +} diff --git a/pkg/multicloud/hcs/keypair.go b/pkg/multicloud/hcs/keypair.go new file mode 100644 index 0000000000..4de3ed5bb0 --- /dev/null +++ b/pkg/multicloud/hcs/keypair.go @@ -0,0 +1,102 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/aokoli/goutils" + "golang.org/x/crypto/ssh" +) + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212676.html +type SKeypair struct { + Fingerprint string `json:"fingerprint"` + Name string `json:"name"` + PublicKey string `json:"public_key"` +} + +func (self *SRegion) getFingerprint(publicKey string) (string, error) { + pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey)) + if err != nil { + return "", fmt.Errorf("publicKey error %s", err) + } + + fingerprint := strings.Replace(ssh.FingerprintLegacyMD5(pk), ":", "", -1) + return fingerprint, nil +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212676.html +func (self *SRegion) GetKeypairs() ([]SKeypair, error) { + ret := []SKeypair{} + return ret, self.list("ecs", "v2.1", "os-keypairs", nil, &ret) +} + +func (self *SRegion) lookUpKeypair(publicKey string) (string, error) { + keypairs, err := self.GetKeypairs() + if err != nil { + return "", err + } + + fingerprint, err := self.getFingerprint(publicKey) + if err != nil { + return "", err + } + + for _, keypair := range keypairs { + if keypair.Fingerprint == fingerprint { + return keypair.Name, nil + } + } + + return "", fmt.Errorf("keypair not found %s", err) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212678.html +func (self *SRegion) ImportKeypair(name, publicKey string) (*SKeypair, error) { + params := map[string]interface{}{ + "keypair": map[string]interface{}{ + "name": name, + "public_key": publicKey, + }, + } + ret := &SKeypair{} + return ret, self.create("ecs", "v2.1", "os-keypairs", params, ret) +} + +func (self *SRegion) importKeypair(publicKey string) (string, error) { + prefix, e := goutils.RandomAlphabetic(6) + if e != nil { + return "", fmt.Errorf("publicKey error %s", e) + } + + name := prefix + strconv.FormatInt(time.Now().Unix(), 10) + k, e := self.ImportKeypair(name, publicKey) + if e != nil { + return "", fmt.Errorf("keypair import error %s", e) + } + return k.Name, nil +} + +func (self *SRegion) syncKeypair(publicKey string) (string, error) { + name, e := self.lookUpKeypair(publicKey) + if e == nil { + return name, nil + } + return self.importKeypair(publicKey) +} diff --git a/pkg/multicloud/hcs/loadbalancer.go b/pkg/multicloud/hcs/loadbalancer.go new file mode 100644 index 0000000000..2892231f7c --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer.go @@ -0,0 +1,601 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "net/url" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +var LB_ALGORITHM_MAP = map[string]string{ + api.LB_SCHEDULER_WRR: "ROUND_ROBIN", + api.LB_SCHEDULER_WLC: "LEAST_CONNECTIONS", + api.LB_SCHEDULER_SCH: "SOURCE_IP", +} + +var LB_PROTOCOL_MAP = map[string]string{ + api.LB_LISTENER_TYPE_HTTP: "HTTP", + api.LB_LISTENER_TYPE_HTTPS: "TERMINATED_HTTPS", + api.LB_LISTENER_TYPE_UDP: "UDP", + api.LB_LISTENER_TYPE_TCP: "TCP", +} + +var LBBG_PROTOCOL_MAP = map[string]string{ + api.LB_LISTENER_TYPE_HTTP: "HTTP", + api.LB_LISTENER_TYPE_HTTPS: "HTTP", + api.LB_LISTENER_TYPE_UDP: "UDP", + api.LB_LISTENER_TYPE_TCP: "TCP", +} + +var LB_STICKY_SESSION_MAP = map[string]string{ + api.LB_STICKY_SESSION_TYPE_INSERT: "HTTP_COOKIE", + api.LB_STICKY_SESSION_TYPE_SERVER: "APP_COOKIE", +} + +var LB_HEALTHCHECK_TYPE_MAP = map[string]string{ + api.LB_HEALTH_CHECK_HTTP: "HTTP", + api.LB_HEALTH_CHECK_TCP: "TCP", + api.LB_HEALTH_CHECK_UDP: "UDP_CONNECT", +} + +type SLoadbalancer struct { + multicloud.SResourceBase + multicloud.HuaweiTags + region *SRegion + subnet *SNetwork + eip *SEip + + Description string `json:"description"` + ProvisioningStatus string `json:"provisioning_status"` + TenantId string `json:"tenant_id"` + ProjectId string `json:"project_id"` + AdminStateUp bool `json:"admin_state_up"` + Provider string `json:"provider"` + Pools []Pool `json:"pools"` + Listeners []Listener `json:"listeners"` + VipPortId string `json:"vip_port_id"` + OperatingStatus string `json:"operating_status"` + VipAddress string `json:"vip_address"` + VipSubnetId string `json:"vip_subnet_id"` + Id string `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type Listener struct { + Id string `json:"id"` +} + +type Pool struct { + Id string `json:"id"` +} + +func (self *SLoadbalancer) GetIEIP() (cloudprovider.ICloudEIP, error) { + if self.GetEip() == nil { + return nil, nil + } + + return self.eip, nil +} + +func (self *SLoadbalancer) GetId() string { + return self.Id +} + +func (self *SLoadbalancer) GetName() string { + return self.Name +} + +func (self *SLoadbalancer) GetGlobalId() string { + return self.Id +} + +func (self *SLoadbalancer) GetStatus() string { + return api.LB_STATUS_ENABLED +} + +func (self *SLoadbalancer) Refresh() error { + lb, err := self.region.GetLoadbalancer(self.GetId()) + if err != nil { + return err + } + + return jsonutils.Update(self, lb) +} + +func (self *SLoadbalancer) IsEmulated() bool { + return false +} + +func (self *SLoadbalancer) GetProjectId() string { + return self.ProjectId +} + +func (self *SLoadbalancer) GetAddress() string { + return self.VipAddress +} + +// todo: api.LB_ADDR_TYPE_INTERNET? +func (self *SLoadbalancer) GetAddressType() string { + return api.LB_ADDR_TYPE_INTRANET +} + +func (self *SLoadbalancer) GetNetworkType() string { + return api.LB_NETWORK_TYPE_VPC +} + +func (self *SLoadbalancer) GetNetworkIds() []string { + net := self.GetNetwork() + if net != nil { + return []string{net.GetId()} + } + + return []string{} +} + +func (self *SLoadbalancer) GetNetwork() *SNetwork { + if self.subnet == nil { + port, err := self.region.GetPort(self.VipPortId) + if err == nil { + net, err := self.region.GetNetwork(port.NetworkId) + if err == nil { + self.subnet = net + } else { + log.Debugf("huawei.SLoadbalancer.getNetwork %s", err) + } + } else { + log.Debugf("huawei.SLoadbalancer.GetPort %s", err) + } + } + + return self.subnet +} + +func (self *SLoadbalancer) GetEip() *SEip { + if self.eip == nil { + eips, _ := self.region.GetEips(self.VipPortId, nil) + for i := range eips { + self.eip = &eips[i] + } + } + return self.eip +} + +func (self *SLoadbalancer) GetVpcId() string { + net := self.GetNetwork() + if net != nil { + return net.VpcId + } + + return "" +} + +func (self *SLoadbalancer) GetZoneId() string { + net := self.GetNetwork() + if net != nil { + z, err := self.region.GetZoneById(net.AvailabilityZone) + if err != nil { + log.Infof("getZoneById %s %s", net.AvailabilityZone, err) + return "" + } + + return z.GetGlobalId() + } + + return "" +} + +func (self *SLoadbalancer) GetZone1Id() string { + return "" +} + +func (self *SLoadbalancer) GetLoadbalancerSpec() string { + return "" +} + +func (self *SLoadbalancer) GetChargeType() string { + eip := self.GetEip() + if eip != nil { + return eip.GetInternetChargeType() + } + + return api.EIP_CHARGE_TYPE_BY_TRAFFIC +} + +func (self *SLoadbalancer) GetEgressMbps() int { + eip := self.GetEip() + if eip != nil { + return eip.GetBandwidth() + } + + return 0 +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0141008275.html +func (self *SLoadbalancer) Delete(ctx context.Context) error { + return self.region.DeleteLoadBalancer(self.GetId()) +} + +func (self *SLoadbalancer) Start() error { + return nil +} + +func (self *SLoadbalancer) Stop() error { + return cloudprovider.ErrNotSupported +} + +func (self *SLoadbalancer) GetILoadBalancerListeners() ([]cloudprovider.ICloudLoadbalancerListener, error) { + ret, err := self.region.GetLoadBalancerListeners(self.GetId()) + if err != nil { + return nil, err + } + + iret := make([]cloudprovider.ICloudLoadbalancerListener, 0) + for i := range ret { + listener := ret[i] + listener.lb = self + iret = append(iret, &listener) + } + + return iret, nil +} + +func (self *SLoadbalancer) GetILoadBalancerBackendGroups() ([]cloudprovider.ICloudLoadbalancerBackendGroup, error) { + ret, err := self.region.GetLoadBalancerBackendGroups(self.GetId()) + if err != nil { + return nil, err + } + + iret := make([]cloudprovider.ICloudLoadbalancerBackendGroup, 0) + for i := range ret { + bg := ret[i] + bg.lb = self + bg.region = self.region + iret = append(iret, &bg) + } + + return iret, nil +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561549.html +func (self *SLoadbalancer) CreateILoadBalancerBackendGroup(group *cloudprovider.SLoadbalancerBackendGroup) (cloudprovider.ICloudLoadbalancerBackendGroup, error) { + ret, err := self.region.CreateLoadBalancerBackendGroup(group) + ret.lb = self + return &ret, err +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561563.html +func (self *SLoadbalancer) CreateHealthCheck(backendGroupId string, healthcheck *cloudprovider.SLoadbalancerHealthCheck) error { + _, err := self.region.CreateLoadBalancerHealthCheck(backendGroupId, healthcheck) + return err +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561548.html +func (self *SLoadbalancer) GetILoadBalancerBackendGroupById(groupId string) (cloudprovider.ICloudLoadbalancerBackendGroup, error) { + ret := &SElbBackendGroup{lb: self, region: self.region} + return ret, self.region.lbGet("lbaas/pools/"+groupId, ret) +} + +func (self *SLoadbalancer) CreateILoadBalancerListener(ctx context.Context, listener *cloudprovider.SLoadbalancerListener) (cloudprovider.ICloudLoadbalancerListener, error) { + ret, err := self.region.CreateLoadBalancerListener(listener) + if err != nil { + return nil, err + } + + ret.lb = self + return &ret, nil +} + +func (self *SLoadbalancer) GetILoadBalancerListenerById(listenerId string) (cloudprovider.ICloudLoadbalancerListener, error) { + ret := &SElbListener{lb: self} + return ret, self.region.lbGet("lbaas/listeners/"+listenerId, ret) +} + +func (self *SRegion) GetLoadbalancer(id string) (*SLoadbalancer, error) { + ret := &SLoadbalancer{region: self} + return ret, self.lbGet("lbaas/loadbalancers/"+id, ret) +} + +func (self *SRegion) DeleteLoadBalancer(elbId string) error { + resource := fmt.Sprintf("lbaas/loadbalancers/%s", elbId) + return self.lbDelete(resource) +} + +func (self *SRegion) GetLoadBalancerListeners(lbId string) ([]SElbListener, error) { + ret := []SElbListener{} + params := url.Values{} + if len(lbId) > 0 { + params.Set("loadbalancer_id", lbId) + } + return ret, self.lbList("lbaas/listeners", params, &ret) +} + +func (self *SRegion) CreateLoadBalancerListener(listener *cloudprovider.SLoadbalancerListener) (SElbListener, error) { + params := map[string]interface{}{ + "name": listener.Name, + "description": listener.Description, + "protocol": LB_PROTOCOL_MAP[listener.ListenerType], + "protocol_port": listener.ListenerPort, + "loadbalancer_id": listener.LoadbalancerID, + "http2_enable": listener.EnableHTTP2, + } + if len(listener.BackendGroupID) > 0 { + params["default_pool_id"] = listener.BackendGroupID + } + + if listener.ListenerType == api.LB_LISTENER_TYPE_HTTPS { + params["default_tls_container_ref"] = listener.CertificateID + } + + if listener.XForwardedFor { + params["insert_headers"] = map[string]interface{}{ + "X-Forwarded-ELB-IP": listener.XForwardedFor, + } + } + + ret := SElbListener{} + err := self.lbCreate("lbaas/listeners", map[string]interface{}{"listener": params}, &ret) + if err != nil { + return ret, err + } + return ret, nil +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561547.html +func (self *SRegion) GetLoadBalancerBackendGroups(elbId string) ([]SElbBackendGroup, error) { + query := url.Values{} + if len(elbId) > 0 { + query.Set("loadbalancer_id", elbId) + } + + ret := []SElbBackendGroup{} + return ret, self.lbList("lbaas/pools", query, &ret) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561547.html +func (self *SRegion) CreateLoadBalancerBackendGroup(group *cloudprovider.SLoadbalancerBackendGroup) (SElbBackendGroup, error) { + ret := SElbBackendGroup{region: self} + var protocol, scheduler string + if s, ok := LB_ALGORITHM_MAP[group.Scheduler]; !ok { + return ret, fmt.Errorf("CreateILoadBalancerBackendGroup unsupported scheduler %s", group.Scheduler) + } else { + scheduler = s + } + + if t, ok := LBBG_PROTOCOL_MAP[group.ListenType]; !ok { + return ret, fmt.Errorf("CreateILoadBalancerBackendGroup unsupported listener type %s", group.ListenType) + } else { + protocol = t + } + + params := map[string]interface{}{ + "project_id": self.client.projectId, + "name": group.Name, + "protocol": protocol, + "lb_algorithm": scheduler, + } + + if len(group.ListenerID) > 0 { + params["listener_id"] = group.ListenerID + } else if len(group.LoadbalancerID) > 0 { + params["loadbalancer_id"] = group.LoadbalancerID + } else { + return ret, fmt.Errorf("CreateLoadBalancerBackendGroup one of listener id / loadbalancer id must be specified") + } + + if group.StickySession != nil { + s := map[string]interface{}{} + timeout := int64(group.StickySession.StickySessionCookieTimeout / 60) + if group.ListenType == api.LB_LISTENER_TYPE_UDP || group.ListenType == api.LB_LISTENER_TYPE_TCP { + s["type"] = "SOURCE_IP" + if timeout > 0 { + s["persistence_timeout"] = timeout + } + } else { + s["type"] = LB_STICKY_SESSION_MAP[group.StickySession.StickySessionType] + if len(group.StickySession.StickySessionCookie) > 0 { + s["cookie_name"] = group.StickySession.StickySessionCookie + } else { + if timeout > 0 { + s["persistence_timeout"] = timeout + } + } + } + params["session_persistence"] = s + } + err := self.lbCreate("lbaas/pools", map[string]interface{}{"pool": params}, &ret) + if err != nil { + return ret, err + } + + if group.HealthCheck != nil { + _, err := self.CreateLoadBalancerHealthCheck(ret.GetId(), group.HealthCheck) + if err != nil { + return ret, err + } + } + + return ret, nil +} + +func (self *SRegion) CreateLoadBalancerHealthCheck(backendGroupId string, healthCheck *cloudprovider.SLoadbalancerHealthCheck) (SElbHealthCheck, error) { + params := map[string]interface{}{ + "delay": healthCheck.HealthCheckInterval, + "max_retries": healthCheck.HealthCheckRise, + "pool_id": backendGroupId, + "timeout": healthCheck.HealthCheckTimeout, + "type": LB_HEALTHCHECK_TYPE_MAP[healthCheck.HealthCheckType], + } + if healthCheck.HealthCheckType == api.LB_HEALTH_CHECK_HTTP { + if len(healthCheck.HealthCheckDomain) > 0 { + params["domain_name"] = healthCheck.HealthCheckDomain + } + + if len(healthCheck.HealthCheckURI) > 0 { + params["url_path"] = healthCheck.HealthCheckURI + } + + if len(healthCheck.HealthCheckHttpCode) > 0 { + params["expected_codes"] = ToHuaweiHealthCheckHttpCode(healthCheck.HealthCheckHttpCode) + } + } + + ret := SElbHealthCheck{region: self} + err := self.lbCreate("lbaas/healthmonitors", map[string]interface{}{"healthmonitor": params}, &ret) + if err != nil { + return ret, err + } + + return ret, nil +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561564.html +func (self *SRegion) UpdateLoadBalancerHealthCheck(healthCheckId string, healthCheck *cloudprovider.SLoadbalancerHealthCheck) (SElbHealthCheck, error) { + params := map[string]interface{}{ + "delay": healthCheck.HealthCheckInterval, + "max_retries": healthCheck.HealthCheckRise, + "timeout": healthCheck.HealthCheckTimeout, + } + if healthCheck.HealthCheckType == api.LB_HEALTH_CHECK_HTTP { + if len(healthCheck.HealthCheckDomain) > 0 { + params["domain_name"] = healthCheck.HealthCheckDomain + } + + if len(healthCheck.HealthCheckURI) > 0 { + params["url_path"] = healthCheck.HealthCheckURI + } + + if len(healthCheck.HealthCheckHttpCode) > 0 { + params["expected_codes"] = ToHuaweiHealthCheckHttpCode(healthCheck.HealthCheckHttpCode) + } + } + + ret := SElbHealthCheck{region: self} + res := "lbaas/healthmonitors/" + healthCheckId + err := self.lbUpdate(res, map[string]interface{}{"healthmonitor": params}) + if err != nil { + return ret, err + } + return ret, self.lbGet(res, &ret) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561565.html +func (self *SRegion) DeleteLoadbalancerHealthCheck(healthCheckId string) error { + return self.lbDelete("lbaas/healthmonitors/" + healthCheckId) +} + +func (self *SLoadbalancer) SetTags(tags map[string]string, replace bool) error { + return cloudprovider.ErrNotSupported +} + +func (self *SRegion) lbGet(resource string, retVal interface{}) error { + return self.client.get("vpc", "v2.0", self.Id, resource, retVal) +} + +func (self *SRegion) lbList(resource string, params url.Values, retVal interface{}) error { + return self.list("vpc", "v2.0", resource, params, retVal) +} + +func (self *SRegion) lbDelete(resource string) error { + return self.delete("vpc", "v2.0", resource) +} + +func (self *SRegion) lbCreate(resource string, params map[string]interface{}, retVal interface{}) error { + return self.create("vpc", "v2.0", resource, params, retVal) +} + +func (self *SRegion) lbUpdate(resource string, params map[string]interface{}) error { + return self.update("vpc", "v2.0", resource, params) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561535.html +func (self *SRegion) CreateLoadBalancer(loadbalancer *cloudprovider.SLoadbalancer) (*SLoadbalancer, error) { + subnet, err := self.GetNetwork(loadbalancer.NetworkIDs[0]) + if err != nil { + return nil, errors.Wrap(err, "SRegion.CreateLoadBalancer.getNetwork") + } + + params := map[string]interface{}{ + "name": loadbalancer.Name, + "vip_subnet_id": subnet.NeutronSubnetId, + "tenant_id": self.client.projectId, + } + if len(loadbalancer.Address) > 0 { + params["vip_address"] = loadbalancer.Address + } + ret := &SLoadbalancer{region: self} + err = self.lbCreate("lbaas/loadbalancers", map[string]interface{}{"loadbalancer": params}, ret) + if err != nil { + return nil, err + } + + // 创建公网类型ELB + if len(loadbalancer.EipID) > 0 { + err := self.AssociateEipWithPortId(loadbalancer.EipID, ret.VipPortId) + if err != nil { + return ret, errors.Wrap(err, "SRegion.CreateLoadBalancer.AssociateEipWithPortId") + } + } + return ret, nil +} + +func (self *SRegion) GetLoadBalancers() ([]SLoadbalancer, error) { + lbs := []SLoadbalancer{} + params := url.Values{} + return lbs, self.lbList("lbaas/loadbalancers", params, &lbs) +} + +func (self *SRegion) GetILoadBalancers() ([]cloudprovider.ICloudLoadbalancer, error) { + elbs, err := self.GetLoadBalancers() + if err != nil { + return nil, err + } + + ret := []cloudprovider.ICloudLoadbalancer{} + for i := range elbs { + elbs[i].region = self + ret = append(ret, &elbs[i]) + } + + return ret, nil +} + +func (self *SRegion) GetILoadBalancerById(id string) (cloudprovider.ICloudLoadbalancer, error) { + elb, err := self.GetLoadbalancer(id) + if err != nil { + return nil, err + } + return elb, nil +} + +func (self *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbalancer) (cloudprovider.ICloudLoadbalancer, error) { + ret, err := self.CreateLoadBalancer(loadbalancer) + if err != nil { + return nil, err + } + + return ret, nil +} diff --git a/pkg/multicloud/hcs/loadbalancer_acl.go b/pkg/multicloud/hcs/loadbalancer_acl.go new file mode 100644 index 0000000000..4fbebf301a --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer_acl.go @@ -0,0 +1,155 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "net/url" + "strings" + + "yunion.io/x/jsonutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SElbACL struct { + multicloud.SResourceBase + multicloud.HuaweiTags + region *SRegion + + Id string `json:"id"` + ListenerId string `json:"listener_id"` + TenantId string `json:"tenant_id"` + EnableWhitelist bool `json:"enable_whitelist"` + Whitelist string `json:"whitelist"` +} + +func (self *SElbACL) GetAclListenerID() string { + return self.ListenerId +} + +func (self *SElbACL) GetId() string { + return self.Id +} + +func (self *SElbACL) GetName() string { + return self.Id +} + +func (self *SElbACL) GetGlobalId() string { + return self.GetId() +} + +func (self *SElbACL) GetStatus() string { + if self.EnableWhitelist { + return api.LB_BOOL_ON + } + + return api.LB_BOOL_OFF +} + +func (self *SElbACL) Refresh() error { + acl, err := self.region.GetLoadBalancerAcl(self.GetId()) + if err != nil { + return err + } + return jsonutils.Update(self, acl) +} + +func (self *SElbACL) IsEmulated() bool { + return false +} + +func (self *SElbACL) GetProjectId() string { + return "" +} + +func (self *SElbACL) GetAclEntries() []cloudprovider.SLoadbalancerAccessControlListEntry { + ret := []cloudprovider.SLoadbalancerAccessControlListEntry{} + for _, cidr := range strings.Split(self.Whitelist, ",") { + ret = append(ret, cloudprovider.SLoadbalancerAccessControlListEntry{CIDR: cidr}) + } + + return ret +} + +func (self *SElbACL) Sync(acl *cloudprovider.SLoadbalancerAccessControlList) error { + whiteList := "" + cidrs := []string{} + for _, entry := range acl.Entrys { + cidrs = append(cidrs, entry.CIDR) + } + whiteList = strings.Join(cidrs, ",") + params := map[string]interface{}{ + "whitelist": whiteList, + "enable_whitelist": acl.AccessControlEnable, + } + return self.region.lbUpdate("lbaas/whitelists/"+self.GetId(), map[string]interface{}{"whitelist": params}) +} + +func (self *SElbACL) Delete() error { + return self.region.lbDelete("lbaas/whitelists/" + self.GetId()) +} + +func (self *SRegion) GetLoadBalancerAcl(aclId string) (*SElbACL, error) { + ret := &SElbACL{region: self} + return ret, self.lbGet("lbaas/whitelists/"+aclId, ret) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561582.html +func (self *SRegion) GetLoadBalancerAcls(listenerId string) ([]SElbACL, error) { + query := url.Values{} + if len(listenerId) > 0 { + query.Set("listener_id", listenerId) + } + ret := []SElbACL{} + return ret, self.lbList("lbaas/whitelists", query, &ret) +} + +func (self *SRegion) CreateLoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (*SElbACL, error) { + params := map[string]interface{}{ + "listener_id": acl.ListenerId, + } + if len(acl.Entrys) > 0 { + whitelist := []string{} + for i := range acl.Entrys { + whitelist = append(whitelist, acl.Entrys[i].CIDR) + } + params["enable_whitelist"] = acl.AccessControlEnable + params["whitelist"] = strings.Join(whitelist, ",") + } else { + params["enable_whitelist"] = false + } + ret := &SElbACL{region: self} + return ret, self.lbCreate("lbaas/whitelists", map[string]interface{}{"whitelist": params}, ret) +} + +func (self *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) { + ret, err := self.CreateLoadBalancerAcl(acl) + if err != nil { + return nil, err + } + + return ret, nil +} + +func (self *SRegion) GetILoadBalancerAclById(aclId string) (cloudprovider.ICloudLoadbalancerAcl, error) { + acl, err := self.GetLoadBalancerAcl(aclId) + if err != nil { + return nil, err + } + return acl, nil +} diff --git a/pkg/multicloud/hcs/loadbalancer_backend.go b/pkg/multicloud/hcs/loadbalancer_backend.go new file mode 100644 index 0000000000..3d26822da3 --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer_backend.go @@ -0,0 +1,141 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SElbBackend struct { + multicloud.SResourceBase + multicloud.HuaweiTags + region *SRegion + lb *SLoadbalancer + backendGroup *SElbBackendGroup + + Name string `json:"name"` + Weight int `json:"weight"` + AdminStateUp bool `json:"admin_state_up"` + SubnetId string `json:"subnet_id"` + TenantId string `json:"tenant_id"` + ProjectId string `json:"project_id"` + Address string `json:"address"` + ProtocolPort int `json:"protocol_port"` + OperatingStatus string `json:"operating_status"` + Id string `json:"id"` +} + +func (self *SElbBackend) GetId() string { + return self.Id +} + +func (self *SElbBackend) GetName() string { + return self.Name +} + +func (self *SElbBackend) GetGlobalId() string { + return self.GetId() +} + +func (self *SElbBackend) GetStatus() string { + return api.LB_STATUS_ENABLED +} + +func (self *SElbBackend) Refresh() error { + backend, err := self.region.GetElbBackend(self.backendGroup.GetId(), self.Id) + if err != nil { + return err + } + return jsonutils.Update(self, backend) +} + +func (self *SElbBackend) IsEmulated() bool { + return false +} + +func (self *SElbBackend) GetProjectId() string { + return "" +} + +func (self *SElbBackend) GetWeight() int { + return self.Weight +} + +func (self *SElbBackend) GetPort() int { + return self.ProtocolPort +} + +func (self *SElbBackend) GetBackendType() string { + return api.LB_BACKEND_GUEST +} + +func (self *SElbBackend) GetBackendRole() string { + return api.LB_BACKEND_ROLE_DEFAULT +} + +func (self *SElbBackend) GetBackendId() string { + i, err := self.lb.region.getInstanceByIP(self.Address) + if err != nil { + log.Errorf("ElbBackend GetBackendId %s", err) + } + + if i != nil { + return i.GetId() + } + + return "" +} + +func (self *SElbBackend) GetIpAddress() string { + return "" +} + +func (self *SElbBackend) SyncConf(ctx context.Context, port, weight int) error { + if port > 0 { + log.Warningf("Elb backend SyncConf unsupport modify port") + } + + params := map[string]interface{}{ + "weight": weight, + } + res := fmt.Sprintf("lbaas/pools/%s/members/%s", self.backendGroup.GetId(), self.Id) + return self.region.lbUpdate(res, map[string]interface{}{"member": params}) +} + +func (self *SRegion) getInstanceByIP(ip string) (*SInstance, error) { + instances, err := self.GetInstances(ip) + if err != nil { + return nil, errors.Wrapf(err, "GetInstance(%s)", ip) + } + for i := range instances { + return &instances[i], nil + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SRegion) GetElbBackend(pool, id string) (*SElbBackend, error) { + res := fmt.Sprintf("lbaas/pools/%s/members/%s", pool, id) + ret := &SElbBackend{region: self} + return ret, self.lbGet(res, ret) +} diff --git a/pkg/multicloud/hcs/loadbalancer_backendgroup.go b/pkg/multicloud/hcs/loadbalancer_backendgroup.go new file mode 100644 index 0000000000..5a71b14b61 --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer_backendgroup.go @@ -0,0 +1,490 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SElbBackendGroup struct { + multicloud.SResourceBase + multicloud.HuaweiTags + lb *SLoadbalancer + region *SRegion + + LBAlgorithm string `json:"lb_algorithm"` + Protocol string `json:"protocol"` + Description string `json:"description"` + AdminStateUp bool `json:"admin_state_up"` + Loadbalancers []Listener `json:"loadbalancers"` + TenantId string `json:"tenant_id"` + ProjectId string `json:"project_id"` + Listeners []Listener `json:"listeners"` + Id string `json:"id"` + Name string `json:"name"` + HealthMonitorId string `json:"healthmonitor_id"` + SessionPersistence StickySession `json:"session_persistence"` +} + +func (self *SElbBackendGroup) GetLoadbalancerId() string { + return self.lb.GetId() +} + +func (self *SElbBackendGroup) GetILoadbalancer() cloudprovider.ICloudLoadbalancer { + return self.lb +} + +type StickySession struct { + Type string `json:"type"` + CookieName string `json:"cookie_name"` + PersistenceTimeout int `json:"persistence_timeout"` +} + +func (self *SElbBackendGroup) GetProtocolType() string { + switch self.Protocol { + case "TCP": + return api.LB_LISTENER_TYPE_TCP + case "UDP": + return api.LB_LISTENER_TYPE_UDP + case "HTTP": + return api.LB_LISTENER_TYPE_HTTP + default: + return "" + } +} + +func (self *SElbBackendGroup) GetScheduler() string { + switch self.LBAlgorithm { + case "ROUND_ROBIN": + return api.LB_SCHEDULER_WRR + case "LEAST_CONNECTIONS": + return api.LB_SCHEDULER_WLC + case "SOURCE_IP": + return api.LB_SCHEDULER_SCH + default: + return "" + } +} + +func ToHuaweiHealthCheckHttpCode(c string) string { + c = strings.TrimSpace(c) + segs := strings.Split(c, ",") + ret := []string{} + for _, seg := range segs { + seg = strings.TrimLeft(seg, "http_") + seg = strings.TrimSpace(seg) + seg = strings.Replace(seg, "xx", "00", -1) + ret = append(ret, seg) + } + + return strings.Join(ret, ",") +} + +func ToOnecloudHealthCheckHttpCode(c string) string { + c = strings.TrimSpace(c) + segs := strings.Split(c, ",") + ret := []string{} + for _, seg := range segs { + seg = strings.TrimSpace(seg) + seg = strings.Replace(seg, "00", "xx", -1) + seg = "http_" + seg + ret = append(ret, seg) + } + + return strings.Join(ret, ",") +} + +func (self *SElbBackendGroup) GetHealthCheck() (*cloudprovider.SLoadbalancerHealthCheck, error) { + if len(self.HealthMonitorId) == 0 { + return nil, nil + } + + health, err := self.region.GetLoadBalancerHealthCheck(self.HealthMonitorId) + if err != nil { + return nil, err + } + + var healthCheckType string + switch health.Type { + case "TCP": + healthCheckType = api.LB_HEALTH_CHECK_TCP + case "UDP_CONNECT": + healthCheckType = api.LB_HEALTH_CHECK_UDP + case "HTTP": + healthCheckType = api.LB_HEALTH_CHECK_HTTP + default: + healthCheckType = "" + } + + ret := cloudprovider.SLoadbalancerHealthCheck{ + HealthCheckType: healthCheckType, + HealthCheckTimeout: health.Timeout, + HealthCheckDomain: health.DomainName, + HealthCheckURI: health.URLPath, + HealthCheckInterval: health.Delay, + HealthCheckRise: health.MaxRetries, + HealthCheckHttpCode: ToOnecloudHealthCheckHttpCode(health.ExpectedCodes), + } + + return &ret, nil +} + +func (self *SElbBackendGroup) GetStickySession() (*cloudprovider.SLoadbalancerStickySession, error) { + if len(self.SessionPersistence.Type) == 0 { + return nil, nil + } + + var stickySessionType string + switch self.SessionPersistence.Type { + case "SOURCE_IP": + stickySessionType = api.LB_STICKY_SESSION_TYPE_INSERT + case "HTTP_COOKIE": + stickySessionType = api.LB_STICKY_SESSION_TYPE_INSERT + case "APP_COOKIE": + stickySessionType = api.LB_STICKY_SESSION_TYPE_SERVER + } + + ret := cloudprovider.SLoadbalancerStickySession{ + StickySession: api.LB_BOOL_ON, + StickySessionCookie: self.SessionPersistence.CookieName, + StickySessionType: stickySessionType, + StickySessionCookieTimeout: self.SessionPersistence.PersistenceTimeout * 60, + } + + return &ret, nil +} + +func (self *SElbBackendGroup) GetId() string { + return self.Id +} + +func (self *SElbBackendGroup) GetName() string { + return self.Name +} + +func (self *SElbBackendGroup) GetGlobalId() string { + return self.GetId() +} + +func (self *SElbBackendGroup) GetStatus() string { + return api.LB_STATUS_ENABLED +} + +func (self *SElbBackendGroup) Refresh() error { + ret, err := self.lb.region.GetLoadBalancerBackendGroupId(self.GetId()) + if err != nil { + return err + } + ret.lb = self.lb + + err = jsonutils.Update(self, ret) + if err != nil { + return err + } + + return nil +} + +func (self *SElbBackendGroup) IsEmulated() bool { + return false +} + +func (self *SElbBackendGroup) GetProjectId() string { + return self.ProjectId +} + +func (self *SElbBackendGroup) IsDefault() bool { + return false +} + +func (self *SElbBackendGroup) GetType() string { + return api.LB_BACKENDGROUP_TYPE_NORMAL +} + +func (self *SElbBackendGroup) GetILoadbalancerBackends() ([]cloudprovider.ICloudLoadbalancerBackend, error) { + ret, err := self.region.GetLoadBalancerBackends(self.GetId()) + if err != nil { + return nil, err + } + + iret := []cloudprovider.ICloudLoadbalancerBackend{} + for i := range ret { + backend := ret[i] + backend.lb = self.lb + backend.backendGroup = self + + iret = append(iret, &backend) + } + + return iret, nil +} + +func (self *SElbBackendGroup) GetILoadbalancerBackendById(serverId string) (cloudprovider.ICloudLoadbalancerBackend, error) { + backend, err := self.region.GetElbBackend(self.GetId(), serverId) + if err != nil { + return nil, err + } + backend.lb = self.lb + backend.backendGroup = self + return backend, nil +} + +func (self *SElbBackendGroup) AddBackendServer(serverId string, weight int, port int) (cloudprovider.ICloudLoadbalancerBackend, error) { + instance, err := self.lb.region.GetInstance(serverId) + if err != nil { + return nil, err + } + + nics, err := instance.GetINics() + if err != nil { + return nil, err + } else if len(nics) == 0 { + return nil, fmt.Errorf("AddBackendServer %s no network interface found", serverId) + } + + subnets, err := self.lb.region.GetInstanceInterfaces(instance.GetId()) + if err != nil { + return nil, err + } else if len(subnets) == 0 { + return nil, fmt.Errorf("AddBackendServer %s no subnet found", serverId) + } + + net, err := self.lb.region.GetNetwork(subnets[0].NetId) + if err != nil { + return nil, err + } + + backend, err := self.region.AddLoadBalancerBackend(self.GetId(), net.NeutronSubnetId, nics[0].GetIP(), port, weight) + if err != nil { + return nil, err + } + + backend.lb = self.lb + backend.backendGroup = self + return backend, nil +} + +func (self *SElbBackendGroup) RemoveBackendServer(backendId string, weight int, port int) error { + ibackend, err := self.GetILoadbalancerBackendById(backendId) + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + return nil + } + + return errors.Wrap(err, "ElbBackendGroup.GetILoadbalancerBackendById") + } + + err = self.region.RemoveLoadBalancerBackend(self.GetId(), backendId) + if err != nil { + return errors.Wrap(err, "ElbBackendGroup.RemoveBackendServer") + } + + return cloudprovider.WaitDeleted(ibackend, 2*time.Second, 30*time.Second) +} + +func (self *SElbBackendGroup) Delete(ctx context.Context) error { + if len(self.HealthMonitorId) > 0 { + err := self.region.DeleteLoadbalancerHealthCheck(self.HealthMonitorId) + if err != nil { + return errors.Wrap(err, "ElbBackendGroup.Delete.DeleteLoadbalancerHealthCheck") + } + } + + // 删除后端服务器组的同时，删除掉无效的后端服务器数据 + { + backends, err := self.region.getLoadBalancerAdminStateDownBackends(self.GetId()) + if err != nil { + return errors.Wrap(err, "SElbBackendGroup.Delete.getLoadBalancerAdminStateDownBackends") + } + + for i := range backends { + backend := backends[i] + err := self.RemoveBackendServer(backend.GetId(), backend.GetPort(), backend.GetWeight()) + if err != nil { + return errors.Wrap(err, "SElbBackendGroup.Delete.RemoveBackendServer") + } + } + } + + err := self.region.DeleteLoadBalancerBackendGroup(self.GetId()) + if err != nil { + return errors.Wrap(err, "ElbBackendGroup.Delete.DeleteLoadBalancerBackendGroup") + } + + return cloudprovider.WaitDeleted(self, 2*time.Second, 30*time.Second) +} + +func (self *SElbBackendGroup) Sync(ctx context.Context, group *cloudprovider.SLoadbalancerBackendGroup) error { + if group == nil { + return nil + } + + _, err := self.region.UpdateLoadBalancerBackendGroup(self.GetId(), group) + return err +} + +func (self *SRegion) GetLoadBalancerBackendGroupId(backendGroupId string) (*SElbBackendGroup, error) { + ret := &SElbBackendGroup{region: self} + res := fmt.Sprintf("lbaas/pools/" + backendGroupId) + return ret, self.lbGet(res, ret) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561550.html +func (self *SRegion) UpdateLoadBalancerBackendGroup(backendGroupId string, group *cloudprovider.SLoadbalancerBackendGroup) (*SElbBackendGroup, error) { + params := map[string]interface{}{ + "name": group.Name, + } + var scheduler string + if s, ok := LB_ALGORITHM_MAP[group.Scheduler]; !ok { + return nil, fmt.Errorf("UpdateLoadBalancerBackendGroup unsupported scheduler %s", group.Scheduler) + } else { + scheduler = s + } + params["lb_algorithm"] = scheduler + + if group.StickySession == nil || group.StickySession.StickySession == api.LB_BOOL_OFF { + params["session_persistence"] = jsonutils.JSONNull + } else { + s := map[string]interface{}{} + timeout := int64(group.StickySession.StickySessionCookieTimeout / 60) + if group.ListenType == api.LB_LISTENER_TYPE_UDP || group.ListenType == api.LB_LISTENER_TYPE_TCP { + s["type"] = "SOURCE_IP" + if timeout > 0 { + s["persistence_timeout"] = timeout + } + } else { + s["type"] = LB_STICKY_SESSION_MAP[group.StickySession.StickySessionType] + if len(group.StickySession.StickySessionCookie) > 0 { + s["cookie_name"] = group.StickySession.StickySessionCookie + } else { + if timeout > 0 { + s["persistence_timeout"] = timeout + } + } + } + params["session_persistence"] = s + } + err := self.lbUpdate("lbaas/pools/"+backendGroupId, map[string]interface{}{"pool": params}) + if err != nil { + return nil, err + } + + ret, err := self.GetLoadBalancerBackendGroupId(backendGroupId) + if err != nil { + return nil, err + } + + if group.HealthCheck == nil && len(ret.HealthMonitorId) > 0 { + err := self.DeleteLoadbalancerHealthCheck(ret.HealthMonitorId) + if err != nil { + return ret, errors.Wrap(err, "DeleteLoadbalancerHealthCheck") + } + } + + if group.HealthCheck != nil { + if len(ret.HealthMonitorId) == 0 { + _, err := self.CreateLoadBalancerHealthCheck(ret.GetId(), group.HealthCheck) + if err != nil { + return ret, errors.Wrap(err, "CreateLoadBalancerHealthCheck") + } + } else { + _, err := self.UpdateLoadBalancerHealthCheck(ret.HealthMonitorId, group.HealthCheck) + if err != nil { + return ret, errors.Wrap(err, "UpdateLoadBalancerHealthCheck") + } + } + } + + ret.region = self + return ret, nil +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561551.html +func (self *SRegion) DeleteLoadBalancerBackendGroup(id string) error { + return self.lbDelete("lbaas/pools/" + id) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561556.html +func (self *SRegion) AddLoadBalancerBackend(backendGroupId, subnetId, ipaddr string, port, weight int) (*SElbBackend, error) { + params := map[string]interface{}{ + "address": ipaddr, + "protocol_port": port, + "subnet_id": subnetId, + "weight": weight, + } + ret := &SElbBackend{} + return ret, self.lbCreate(fmt.Sprintf("lbaas/pools/%s/members", backendGroupId), map[string]interface{}{"member": params}, ret) +} + +func (self *SRegion) RemoveLoadBalancerBackend(lbbgId string, backendId string) error { + return self.lbDelete(fmt.Sprintf("lbaas/pools/%s/members/%s", lbbgId, backendId)) +} + +func (self *SRegion) getLoadBalancerBackends(backendGroupId string) ([]SElbBackend, error) { + res := fmt.Sprintf("lbaas/pools/%s/members", backendGroupId) + ret := []SElbBackend{} + return ret, self.lbList(res, url.Values{}, &ret) +} + +func (self *SRegion) GetLoadBalancerBackends(backendGroupId string) ([]SElbBackend, error) { + ret, err := self.getLoadBalancerBackends(backendGroupId) + if err != nil { + return nil, errors.Wrap(err, "SRegion.GetLoadBalancerBackends.getLoadBalancerBackends") + } + + // 过滤掉服务器已经被删除的backend。原因是运管平台查询不到已删除的服务器记录，导致同步出错。产生肮数据。 + filtedRet := []SElbBackend{} + for i := range ret { + if ret[i].AdminStateUp { + backend := ret[i] + filtedRet = append(filtedRet, backend) + } + } + + return filtedRet, nil +} + +func (self *SRegion) getLoadBalancerAdminStateDownBackends(backendGroupId string) ([]SElbBackend, error) { + ret, err := self.getLoadBalancerBackends(backendGroupId) + if err != nil { + return nil, errors.Wrap(err, "SRegion.getLoadBalancerAdminStateDownBackends.getLoadBalancerBackends") + } + + filtedRet := []SElbBackend{} + for i := range ret { + if !ret[i].AdminStateUp { + backend := ret[i] + filtedRet = append(filtedRet, backend) + } + } + + return filtedRet, nil +} + +func (self *SRegion) GetLoadBalancerHealthCheck(healthCheckId string) (*SElbHealthCheck, error) { + ret := &SElbHealthCheck{region: self} + return ret, self.lbGet("lbaas/healthmonitors/"+healthCheckId, ret) +} diff --git a/pkg/multicloud/hcs/loadbalancer_cert.go b/pkg/multicloud/hcs/loadbalancer_cert.go new file mode 100644 index 0000000000..c98ff1d66f --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer_cert.go @@ -0,0 +1,182 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "crypto/sha1" + "fmt" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SElbCert struct { + multicloud.SResourceBase + multicloud.HcsTags + region *SRegion + + Certificate string `json:"certificate"` + CreateTime time.Time `json:"create_time"` + ExpireTime time.Time `json:"expire_time"` + Description string `json:"description"` + Domain string `json:"domain"` + ID string `json:"id"` + AdminStateUp bool `json:"admin_state_up"` + TenantID string `json:"tenant_id"` + Name string `json:"name"` + PrivateKey string `json:"private_key"` + Type string `json:"type"` + UpdateTime time.Time `json:"update_time"` +} + +func (self *SElbCert) GetPublickKey() string { + return self.Certificate +} + +func (self *SElbCert) GetPrivateKey() string { + return self.PrivateKey +} + +func (self *SElbCert) GetId() string { + return self.ID +} + +func (self *SElbCert) GetName() string { + return self.Name +} + +func (self *SElbCert) GetGlobalId() string { + return self.GetId() +} + +func (self *SElbCert) GetStatus() string { + return api.LB_STATUS_ENABLED +} + +func (self *SElbCert) Refresh() error { + cert, err := self.region.GetLoadBalancerCertificate(self.GetId()) + if err != nil { + return err + } + return jsonutils.Update(self, cert) +} + +func (self *SElbCert) IsEmulated() bool { + return false +} + +func (self *SElbCert) GetProjectId() string { + return "" +} + +func (self *SElbCert) Sync(name, privateKey, publickKey string) error { + params := map[string]interface{}{ + "name": name, + "private_key": privateKey, + "certificate": publickKey, + } + return self.region.lbUpdate("lbaas/certificates/"+self.GetId(), params) +} + +func (self *SElbCert) Delete() error { + return self.region.lbDelete("lbaas/certificates/" + self.GetId()) +} + +func (self *SElbCert) GetCommonName() string { + return self.Domain +} + +func (self *SElbCert) GetSubjectAlternativeNames() string { + return self.Domain +} + +func (self *SElbCert) GetFingerprint() string { + _fp := sha1.Sum([]byte(self.Certificate)) + fp := fmt.Sprintf("sha1:% x", _fp) + return strings.Replace(fp, " ", ":", -1) +} + +func (self *SElbCert) GetExpireTime() time.Time { + return self.ExpireTime +} + +func (self *SRegion) GetLoadBalancerCertificate(id string) (*SElbCert, error) { + ret := &SElbCert{region: self} + err := self.lbGet("lbaas/certificates/"+id, ret) + if err != nil { + return nil, err + } + return ret, nil +} + +// https://support.huaweicloud.com/api-elb/elb_qy_zs_0001.html +func (self *SRegion) CreateLoadBalancerCertificate(cert *cloudprovider.SLoadbalancerCertificate) (*SElbCert, error) { + params := map[string]interface{}{ + "name": cert.Name, + "private_key": cert.PrivateKey, + "certificate": cert.Certificate, + } + ret := &SElbCert{region: self} + err := self.lbCreate("lbaas/certificates", params, ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (self *SRegion) GetLoadBalancerCertificates() ([]SElbCert, error) { + ret := []SElbCert{} + err := self.lbList("lbaas/certificates", url.Values{}, &ret) + if err != nil { + return nil, err + } + return ret, nil +} + +func (self *SRegion) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoadbalancerCertificate, error) { + ret, err := self.GetLoadBalancerCertificates() + if err != nil { + return nil, err + } + + iret := make([]cloudprovider.ICloudLoadbalancerCertificate, len(ret)) + for i := range ret { + ret[i].region = self + iret[i] = &ret[i] + } + return iret, nil +} + +func (self *SRegion) CreateILoadBalancerCertificate(cert *cloudprovider.SLoadbalancerCertificate) (cloudprovider.ICloudLoadbalancerCertificate, error) { + ret, err := self.CreateLoadBalancerCertificate(cert) + if err != nil { + return nil, err + } + return ret, nil +} + +func (self *SRegion) GetILoadBalancerCertificateById(certId string) (cloudprovider.ICloudLoadbalancerCertificate, error) { + cert, err := self.GetLoadBalancerCertificate(certId) + if err != nil { + return nil, err + } + return cert, nil +} diff --git a/pkg/multicloud/hcs/loadbalancer_healthcheck.go b/pkg/multicloud/hcs/loadbalancer_healthcheck.go new file mode 100644 index 0000000000..cd353bbeef --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer_healthcheck.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +type SElbHealthCheck struct { + region *SRegion + + Name string `json:"name"` + AdminStateUp bool `json:"admin_state_up"` + TenantId string `json:"tenant_id"` + ProjectId string `json:"project_id"` + DomainName string `json:"domain_name"` + Delay int `json:"delay"` + ExpectedCodes string `json:"expected_codes"` + MaxRetries int `json:"max_retries"` + HTTPMethod string `json:"http_method"` + Timeout int `json:"timeout"` + Pools []Pool `json:"pools"` + URLPath string `json:"url_path"` + Type string `json:"type"` + Id string `json:"id"` + MonitorPort int `json:"monitor_port"` +} diff --git a/pkg/multicloud/hcs/loadbalancer_listener.go b/pkg/multicloud/hcs/loadbalancer_listener.go new file mode 100644 index 0000000000..712bedf242 --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer_listener.go @@ -0,0 +1,643 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "net/url" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type InsertHeaders struct { + XForwardedELBIP bool `json:"X-Forwarded-ELB-IP"` +} + +type Loadbalancer struct { + Id string `json:"id"` +} + +type SElbListener struct { + multicloud.SResourceBase + multicloud.SLoadbalancerRedirectBase + multicloud.HuaweiTags + lb *SLoadbalancer + acl *SElbACL + backendgroup *SElbBackendGroup + + ProtocolPort int `json:"protocol_port"` + Protocol string `json:"protocol"` + Description string `json:"description"` + AdminStateUp bool `json:"admin_state_up"` + Http2Enable bool `json:"http2_enable"` + Loadbalancers []Loadbalancer `json:"loadbalancers"` + TenantId string `json:"tenant_id"` + ProjectId string `json:"project_id"` + ConnectionLimit int `json:"connection_limit"` + DefaultPoolId string `json:"default_pool_id"` + Id string `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + InsertHeaders InsertHeaders `json:"insert_headers"` + DefaultTlsContainerRef string `json:"default_tls_container_ref"` +} + +func (self *SElbListener) GetId() string { + return self.Id +} + +func (self *SElbListener) GetName() string { + return self.Name +} + +func (self *SElbListener) GetGlobalId() string { + return self.GetId() +} + +func (self *SElbListener) GetStatus() string { + return api.LB_STATUS_ENABLED +} + +func (self *SElbListener) Refresh() error { + ilistener, err := self.lb.GetILoadBalancerListenerById(self.GetId()) + if err != nil { + return err + } + + listener := ilistener.(*SElbListener) + listener.lb = self.lb + err = jsonutils.Update(self, listener) + if err != nil { + return err + } + + return nil +} + +func (self *SElbListener) IsEmulated() bool { + return false +} + +func (self *SElbListener) GetProjectId() string { + return self.ProjectId +} + +func (self *SElbListener) GetListenerType() string { + switch self.Protocol { + case "TCP": + return api.LB_LISTENER_TYPE_TCP + case "UDP": + return api.LB_LISTENER_TYPE_UDP + case "HTTP": + return api.LB_LISTENER_TYPE_HTTP + case "TERMINATED_HTTPS": + return api.LB_LISTENER_TYPE_HTTPS + case "HTTPS": + return api.LB_LISTENER_TYPE_HTTPS + default: + return "" + } +} + +func (self *SElbListener) GetListenerPort() int { + return self.ProtocolPort +} + +func (self *SElbListener) GetBackendGroup() (*SElbBackendGroup, error) { + if self.backendgroup == nil { + lbbgId := self.GetBackendGroupId() + if len(lbbgId) > 0 { + lbbg, err := self.lb.GetILoadBalancerBackendGroupById(lbbgId) + if err != nil { + return nil, err + } + + self.backendgroup = lbbg.(*SElbBackendGroup) + } + } + + return self.backendgroup, nil +} + +func (self *SElbListener) GetScheduler() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetScheduler %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + return lbbg.GetScheduler() +} + +func (self *SElbListener) GetAcl() (*SElbACL, error) { + if self.acl != nil { + return self.acl, nil + } + + acls, err := self.lb.region.GetLoadBalancerAcls(self.GetId()) + if err != nil { + return nil, err + } + + if len(acls) == 0 { + return nil, nil + } else { + self.acl = &acls[0] + return &acls[0], nil + } +} + +func (self *SElbListener) GetAclStatus() string { + acl, err := self.GetAcl() + if err != nil { + log.Debugf("GetAclStatus %s", err) + return "" + } + + if acl != nil && acl.EnableWhitelist { + return api.LB_BOOL_ON + } + + return api.LB_BOOL_OFF +} + +func (self *SElbListener) GetAclType() string { + return api.LB_ACL_TYPE_WHITE +} + +func (self *SElbListener) GetAclId() string { + acl, err := self.GetAcl() + if err != nil { + log.Debugf("GetAclStatus %s", err) + return "" + } + + if acl == nil { + return "" + } + + return acl.GetId() +} + +func (self *SElbListener) GetEgressMbps() int { + return 0 +} + +func (self *SElbListener) GetHealthCheck() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetHealthCheck %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + health, err := lbbg.GetHealthCheck() + if err != nil { + log.Errorf("ElbListener GetHealthCheck %s", err.Error()) + } + + if health != nil { + return api.LB_BOOL_ON + } else { + return api.LB_BOOL_OFF + } +} + +func (self *SElbListener) GetHealthCheckType() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetHealthCheckType %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + health, err := lbbg.GetHealthCheck() + if err != nil { + log.Errorf("ElbListener GetHealthCheckType %s", err.Error()) + } + + if health != nil { + return health.HealthCheckType + } + + return "" +} + +func (self *SElbListener) GetHealthCheckTimeout() int { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetHealthCheckTimeout %s", err.Error()) + } + + if lbbg == nil { + return 0 + } + + health, err := lbbg.GetHealthCheck() + if err != nil { + log.Errorf("ElbListener GetHealthCheckTimeout %s", err.Error()) + } + + if health != nil { + return health.HealthCheckTimeout + } + + return 0 +} + +func (self *SElbListener) GetHealthCheckInterval() int { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetHealthCheckInterval %s", err.Error()) + } + + if lbbg == nil { + return 0 + } + + health, err := lbbg.GetHealthCheck() + if err != nil { + log.Errorf("ElbListener GetHealthCheckInterval %s", err.Error()) + } + + if health != nil { + return health.HealthCheckInterval + } + + return 0 +} + +func (self *SElbListener) GetHealthCheckRise() int { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetHealthCheckRise %s", err.Error()) + } + + if lbbg == nil { + return 0 + } + + health, err := lbbg.GetHealthCheck() + if err != nil { + log.Errorf("ElbListener GetHealthCheckRise %s", err.Error()) + } + + if health != nil { + return health.HealthCheckRise + } else { + return 0 + } +} + +func (self *SElbListener) GetHealthCheckFail() int { + return 0 +} + +func (self *SElbListener) GetHealthCheckReq() string { + return "" +} + +func (self *SElbListener) GetHealthCheckExp() string { + return "" +} + +func (self *SElbListener) GetBackendGroupId() string { + return self.DefaultPoolId +} + +func (self *SElbListener) GetBackendServerPort() int { + return 0 +} + +func (self *SElbListener) GetHealthCheckDomain() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetHealthCheckDomain %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + health, err := lbbg.GetHealthCheck() + if err != nil { + log.Errorf("ElbListener GetHealthCheckDomain %s", err.Error()) + } + + if health != nil { + return health.HealthCheckDomain + } + + return "" +} + +func (self *SElbListener) GetHealthCheckURI() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetHealthCheckURI %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + health, err := lbbg.GetHealthCheck() + if err != nil { + log.Errorf("ElbListener GetHealthCheckURI %s", err.Error()) + } + + if health != nil { + return health.HealthCheckURI + } + + return "" +} + +func (self *SElbListener) GetHealthCheckCode() string { + return "" +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0136295317.html +func (self *SElbListener) CreateILoadBalancerListenerRule(rule *cloudprovider.SLoadbalancerListenerRule) (cloudprovider.ICloudLoadbalancerListenerRule, error) { + l7policy, err := self.lb.region.CreateLoadBalancerPolicy(self.GetId(), rule) + if err != nil { + return nil, err + } + + l7policy.region = self.lb.region + l7policy.lb = self.lb + l7policy.listener = self + return l7policy, nil +} + +func (self *SElbListener) GetILoadBalancerListenerRuleById(ruleId string) (cloudprovider.ICloudLoadbalancerListenerRule, error) { + ret := &SElbListenerPolicy{region: self.lb.region, lb: self.lb, listener: self} + return ret, self.lb.region.lbGet("lbaas/l7policies/"+ruleId, ret) +} + +func (self *SElbListener) GetILoadbalancerListenerRules() ([]cloudprovider.ICloudLoadbalancerListenerRule, error) { + ret, err := self.lb.region.GetLoadBalancerPolicies(self.GetId()) + if err != nil { + return nil, err + } + + iret := []cloudprovider.ICloudLoadbalancerListenerRule{} + for i := range ret { + rule := ret[i] + rule.listener = self + rule.lb = self.lb + rule.region = self.lb.region + iret = append(iret, &rule) + } + + return iret, nil +} + +func (self *SElbListener) GetStickySession() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetStickySession %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + stickySession, err := lbbg.GetStickySession() + if err != nil { + log.Errorf("ElbListener GetStickySession %s", err.Error()) + } + + if stickySession != nil { + return stickySession.StickySession + } + + return "" +} + +func (self *SElbListener) GetStickySessionType() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetStickySessionType %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + stickySession, err := lbbg.GetStickySession() + if err != nil { + log.Errorf("ElbListener GetStickySessionType %s", err.Error()) + } + + if stickySession != nil { + return stickySession.StickySessionType + } + + return "" +} + +func (self *SElbListener) GetStickySessionCookie() string { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetStickySessionCookie %s", err.Error()) + } + + if lbbg == nil { + return "" + } + + stickySession, err := lbbg.GetStickySession() + if err != nil { + log.Errorf("ElbListener GetStickySessionCookie %s", err.Error()) + } + + if stickySession != nil { + return stickySession.StickySessionCookie + } + + return "" +} + +func (self *SElbListener) GetStickySessionCookieTimeout() int { + lbbg, err := self.GetBackendGroup() + if err != nil { + log.Errorf("ElbListener GetStickySessionCookieTimeout %s", err.Error()) + } + + if lbbg == nil { + return 0 + } + + stickySession, err := lbbg.GetStickySession() + if err != nil { + log.Errorf("ElbListener GetStickySessionCookieTimeout %s", err.Error()) + } + + if stickySession != nil { + return stickySession.StickySessionCookieTimeout + } + + return 0 +} + +func (self *SElbListener) XForwardedForEnabled() bool { + return self.InsertHeaders.XForwardedELBIP +} + +func (self *SElbListener) GzipEnabled() bool { + return false +} + +func (self *SElbListener) GetCertificateId() string { + return self.DefaultTlsContainerRef +} + +func (self *SElbListener) GetTLSCipherPolicy() string { + return "" +} + +func (self *SElbListener) HTTP2Enabled() bool { + return self.Http2Enable +} + +func (self *SElbListener) Start() error { + return nil +} + +func (self *SElbListener) Stop() error { + return cloudprovider.ErrNotSupported +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0096561544.html +/* +default_pool_id有如下限制： +不能更新为其他监听器的default_pool。 +不能更新为其他监听器的关联的转发策略所使用的pool。 +default_pool_id对应的后端云服务器组的protocol和监听器的protocol有如下关系： +监听器的protocol为TCP时，后端云服务器组的protocol必须为TCP。 +监听器的protocol为UDP时，后端云服务器组的protocol必须为UDP。 +监听器的protocol为HTTP或TERMINATED_HTTPS时，后端云服务器组的protocol必须为HTTP。 +*/ +func (self *SElbListener) Sync(ctx context.Context, listener *cloudprovider.SLoadbalancerListener) error { + return self.lb.region.UpdateLoadBalancerListener(self.GetId(), listener) +} + +func (self *SElbListener) Delete(ctx context.Context) error { + return self.lb.region.lbDelete("lbaas/listeners/" + self.GetId()) +} + +func (self *SRegion) UpdateLoadBalancerListener(listenerId string, listener *cloudprovider.SLoadbalancerListener) error { + params := map[string]interface{}{ + "name": listener.Name, + "description": listener.Description, + "http2_enable": listener.EnableHTTP2, + "default_pool_id": jsonutils.JSONNull, + } + if len(listener.BackendGroupID) > 0 { + params["default_pool_id"] = listener.BackendGroupID + } + + if listener.ListenerType == api.LB_LISTENER_TYPE_HTTPS { + params["default_tls_container_ref"] = listener.CertificateID + } + + if listener.XForwardedFor { + params["insert_headers"] = map[string]interface{}{ + "X-Forwarded-ELB-IP": listener.XForwardedFor, + } + } + return self.lbUpdate("elb/listeners/"+listenerId, map[string]interface{}{"listener": params}) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0136295315.html +func (self *SRegion) GetLoadBalancerPolicies(listenerId string) ([]SElbListenerPolicy, error) { + query := url.Values{} + if len(listenerId) > 0 { + query.Set("listener_id", listenerId) + } + + ret := []SElbListenerPolicy{} + return ret, self.lbList("lbaas/l7policies", query, &ret) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0116649234.html +func (self *SRegion) GetLoadBalancerPolicyRules(policyId string) ([]SElbListenerPolicyRule, error) { + ret := []SElbListenerPolicyRule{} + return ret, self.lbList(fmt.Sprintf("lbaas/l7policies/%s/rules", policyId), url.Values{}, &ret) +} + +// https://support.huaweicloud.com/api-elb/zh-cn_topic_0136295317.html +func (self *SRegion) CreateLoadBalancerPolicy(listenerId string, rule *cloudprovider.SLoadbalancerListenerRule) (*SElbListenerPolicy, error) { + ret := &SElbListenerPolicy{} + params := map[string]interface{}{ + "name": rule.Name, + "listener_id": listenerId, + "action": "REDIRECT_TO_POOL", + "redirect_pool_id": rule.BackendGroupID, + } + err := self.lbCreate("lbaas/l7policies", map[string]interface{}{"l7policy": params}, ret) + if err != nil { + return nil, err + } + if len(rule.Domain) > 0 { + params := map[string]interface{}{ + "type": "HOST_NAME", + "value": rule.Domain, + "compare_type": "EQUAL_TO", + } + err := self.lbCreate(fmt.Sprintf("lbaas/l7policies/%s/rules", ret.GetId()), map[string]interface{}{"rule": params}, nil) + if err != nil { + return ret, err + } + } + + if len(rule.Path) > 0 { + params := map[string]interface{}{ + "type": "PATH", + "value": rule.Path, + "compare_type": "EQUAL_TO", + } + err := self.lbCreate(fmt.Sprintf("lbaas/l7policies/%s/rules", ret.GetId()), map[string]interface{}{"rule": params}, nil) + if err != nil { + return ret, err + } + } + + return ret, nil +} + +func (self *SElbListener) GetClientIdleTimeout() int { + return 0 +} + +func (self *SElbListener) GetBackendConnectTimeout() int { + return 0 +} diff --git a/pkg/multicloud/hcs/loadbalancer_listener_rule.go b/pkg/multicloud/hcs/loadbalancer_listener_rule.go new file mode 100644 index 0000000000..65e6d80775 --- /dev/null +++ b/pkg/multicloud/hcs/loadbalancer_listener_rule.go @@ -0,0 +1,168 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SElbListenerPolicy struct { + multicloud.SResourceBase + multicloud.SLoadbalancerRedirectBase + multicloud.HuaweiTags + region *SRegion + lb *SLoadbalancer + listener *SElbListener + + RedirectPoolId string `json:"redirect_pool_id"` + RedirectListenerId *string `json:"redirect_listener_id"` + Description string `json:"description"` + AdminStateUp bool `json:"admin_state_up"` + Rules []Rule `json:"rules"` + TenantId string `json:"tenant_id"` + ProjectId string `json:"project_id"` + ListenerId string `json:"listener_id"` + RedirectURL *string `json:"redirect_url"` + ProvisioningStatus string `json:"provisioning_status"` + Action string `json:"action"` + Position int64 `json:"position"` + Id string `json:"id"` + Name string `json:"name"` +} + +type Rule struct { + Id string `json:"id"` +} + +type SElbListenerPolicyRule struct { + region *SRegion + policy *SElbListenerPolicy + + CompareType string `json:"compare_type"` + ProvisioningStatus string `json:"provisioning_status"` + AdminStateUp bool `json:"admin_state_up"` + TenantId string `json:"tenant_id"` + ProjectId string `json:"project_id"` + Invert bool `json:"invert"` + Value string `json:"value"` + Key interface{} `json:"key"` + Type string `json:"type"` + Id string `json:"id"` +} + +func (self *SElbListenerPolicy) GetId() string { + return self.Id +} + +func (self *SElbListenerPolicy) GetName() string { + return self.Name +} + +func (self *SElbListenerPolicy) GetGlobalId() string { + return self.GetId() +} + +// 负载均衡没有启用禁用操作 +func (self *SElbListenerPolicy) GetStatus() string { + return api.LB_STATUS_ENABLED +} + +func (self *SElbListenerPolicy) Refresh() error { + res := "lbaas/l7policies/" + self.Id + ret := &SElbListenerPolicy{} + err := self.lb.region.lbGet(res, ret) + if err != nil { + return err + } + return jsonutils.Update(self, ret) +} + +func (self *SElbListenerPolicy) IsDefault() bool { + return false +} + +func (self *SElbListenerPolicy) IsEmulated() bool { + return false +} + +func (self *SElbListenerPolicy) GetProjectId() string { + return "" +} + +func (self *SElbListenerPolicy) GetRules() ([]SElbListenerPolicyRule, error) { + ret, err := self.region.GetLoadBalancerPolicyRules(self.GetId()) + if err != nil { + return nil, err + } + + for i := range ret { + ret[i].region = self.lb.region + ret[i].policy = self + } + + return ret, nil +} + +func (self *SElbListenerPolicy) GetDomain() string { + rules, err := self.GetRules() + if err != nil { + log.Errorf("loadbalancer rule GetDomain %s", err) + } + + for i := range rules { + if rules[i].Type == "HOST_NAME" { + return rules[i].Value + } + } + + return "" +} + +func (self *SElbListenerPolicy) GetCondition() string { + return "" +} + +func (self *SElbListenerPolicy) GetPath() string { + rules, err := self.GetRules() + if err != nil { + log.Errorf("loadbalancer rule GetPath %s", err) + } + + for i := range rules { + if rules[i].Type == "PATH" { + return rules[i].Value + } + } + + return "" +} + +func (self *SElbListenerPolicy) GetBackendGroupId() string { + return self.RedirectPoolId +} + +func (self *SElbListenerPolicy) Delete(ctx context.Context) error { + return self.region.DeleteLoadBalancerPolicy(self.GetId()) +} + +func (self *SRegion) DeleteLoadBalancerPolicy(policyId string) error { + return self.lbDelete("lbaas/l7policies/" + policyId) +} diff --git a/pkg/multicloud/hcs/monitor.go b/pkg/multicloud/hcs/monitor.go new file mode 100644 index 0000000000..b6d690a2d1 --- /dev/null +++ b/pkg/multicloud/hcs/monitor.go @@ -0,0 +1,494 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +type MetricData struct { + Namespace string + MetricName string + Dimensions []struct { + Name string + Value string + } + Datapoints []struct { + Average float64 + Timestamp int64 + } + Unit string +} + +func (self *SHcsClient) monitorPost(resource string, params map[string]interface{}) (jsonutils.JSONObject, error) { + url := self._url("ces", "V1.0", self.defaultRegion, resource) + return self.request(httputils.POST, url, nil, params) +} + +func (self *SHcsClient) getServerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + params := map[string]interface{}{ + "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), + "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), + "period": "1", + "filter": "average", + } + metrics := []interface{}{} + namespace, dimesionName, metricNames := "SYS.ECS", "instance_id", []string{ + "cpu_util", + "network_incoming_bytes_aggregate_rate", + "network_outgoing_bytes_aggregate_rate", + "disk_read_bytes_rate", + "disk_write_bytes_rate", + "disk_read_requests_rate", + "disk_write_requests_rate", + } + for _, metricName := range metricNames { + metrics = append(metrics, map[string]interface{}{ + "namespace": namespace, + "metric_name": metricName, + "dimensions": []map[string]string{ + { + "name": dimesionName, + "value": opts.ResourceId, + }, + }, + }) + } + params["metrics"] = metrics + resp, err := self.monitorPost("batch-query-metric-data", params) + if err != nil { + return nil, err + } + metricData := []MetricData{} + err = resp.Unmarshal(&metricData, "metrics") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + result := []cloudprovider.MetricValues{} + for i := range metricData { + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData[i].Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData[i].MetricName { + case "cpu_util": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_CPU_USAGE + case "network_incoming_bytes_aggregate_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_RX + tags = map[string]string{"net_type": "internet"} + case "network_outgoing_bytes_aggregate_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_NET_BPS_TX + tags = map[string]string{"net_type": "internet"} + case "disk_read_bytes_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_BPS + case "disk_write_bytes_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_BPS + case "disk_read_requests_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_READ_IOPS + case "disk_write_requests_rate": + ret.MetricType = cloudprovider.VM_METRIC_TYPE_DISK_IO_WRITE_IOPS + default: + log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData[i].Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + +func (self *SHcsClient) getRedisMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + params := map[string]interface{}{ + "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), + "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), + "period": "1", + "filter": "average", + } + metrics := []interface{}{} + namespace, dimesionName, metricNames := "SYS.DCS", "dcs_instance_id", []string{ + "cpu_usage", + "memory_usage", + "instantaneous_input_kbps", + "instantaneous_output_kbps", + "connected_clients", + "instantaneous_ops", + "keys", + "expires", + "used_memory_dataset", + } + for _, metricName := range metricNames { + metrics = append(metrics, map[string]interface{}{ + "namespace": namespace, + "metric_name": metricName, + "dimensions": []map[string]string{ + { + "name": dimesionName, + "value": opts.ResourceId, + }, + }, + }) + } + params["metrics"] = metrics + resp, err := self.monitorPost("batch-query-metric-data", params) + if err != nil { + return nil, err + } + metricData := []MetricData{} + err = resp.Unmarshal(&metricData, "metrics") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + result := []cloudprovider.MetricValues{} + for i := range metricData { + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData[i].Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData[i].MetricName { + case "cpu_usage": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_CPU_USAGE + case "memory_usage": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_MEM_USAGE + case "instantaneous_input_kbps": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_NET_BPS_RX + case "instantaneous_output_kbps": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_NET_BPS_TX + case "connected_clients": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_USED_CONN + case "instantaneous_ops": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_OPT_SES + case "keys": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_CACHE_KEYS + case "expires": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_CACHE_EXP_KEYS + case "used_memory_dataset": + ret.MetricType = cloudprovider.REDIS_METRIC_TYPE_DATA_MEM_USAGE + default: + log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData[i].Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + +func (self *SHcsClient) getRdsMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + params := map[string]interface{}{ + "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), + "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), + "period": "1", + "filter": "average", + } + metrics := []interface{}{} + namespace, dimesionName, metricNames := "SYS.RDS", "rds_cluster_id", []string{ + "rds001_cpu_util", + "rds002_mem_util", + "rds004_bytes_in", + "rds004_bytes_in", + "rds005_bytes_out", + "rds039_disk_util", + "rds049_disk_read_throughput", + "rds050_disk_write_throughput", + "rds006_conn_count", + "rds008_qps", + "rds009_tps", + "rds013_innodb_reads", + "rds014_innodb_writes", + } + switch opts.Engine { + case api.DBINSTANCE_TYPE_POSTGRESQL: + dimesionName = "postgresql_cluster_id" + case api.DBINSTANCE_TYPE_SQLSERVER: + dimesionName = "rds_cluster_sqlserver_id" + } + + for _, metricName := range metricNames { + metrics = append(metrics, map[string]interface{}{ + "namespace": namespace, + "metric_name": metricName, + "dimensions": []map[string]string{ + { + "name": dimesionName, + "value": opts.ResourceId, + }, + }, + }) + } + params["metrics"] = metrics + resp, err := self.monitorPost("batch-query-metric-data", params) + if err != nil { + return nil, err + } + metricData := []MetricData{} + err = resp.Unmarshal(&metricData, "metrics") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + result := []cloudprovider.MetricValues{} + for i := range metricData { + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData[i].Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData[i].MetricName { + case "rds001_cpu_util": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CPU_USAGE + case "rds002_mem_util": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_MEM_USAGE + case "rds004_bytes_in": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_RX + case "rds005_bytes_out": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_NET_BPS_TX + case "rds039_disk_util": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_USAGE + case "rds049_disk_read_throughput": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_READ_BPS + case "rds050_disk_write_throughput": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_DISK_WRITE_BPS + case "rds006_conn_count": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_CONN_COUNT + case "rds008_qps": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_QPS + case "rds009_tps": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_TPS + case "rds013_innodb_reads": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_READ_BPS + case "rds014_innodb_writes": + ret.MetricType = cloudprovider.RDS_METRIC_TYPE_INNODB_WRITE_BPS + default: + log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData[i].Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + +func (self *SHcsClient) getBucketMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + params := map[string]interface{}{ + "from": opts.StartTime.UnixMilli(), + "to": opts.EndTime.UnixMilli(), + "period": "1", + "filter": "average", + } + metrics := []interface{}{} + namespace, dimesionName, metricNames := "SYS.OBS", "bucket_name", []string{ + "download_bytes", + "upload_bytes", + "first_byte_latency", + "get_request_count", + "request_count_4xx", + "request_count_5xx", + } + + for _, metricName := range metricNames { + metrics = append(metrics, map[string]interface{}{ + "namespace": namespace, + "metric_name": metricName, + "dimensions": []map[string]string{ + { + "name": dimesionName, + "value": opts.ResourceId, + }, + }, + }) + } + params["metrics"] = metrics + resp, err := self.monitorPost("batch-query-metric-data", params) + if err != nil { + return nil, err + } + metricData := []MetricData{} + err = resp.Unmarshal(&metricData, "metrics") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + result := []cloudprovider.MetricValues{} + for i := range metricData { + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData[i].Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData[i].MetricName { + case "download_bytes": + ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_TX + case "upload_bytes": + ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_NET_BPS_RX + case "first_byte_latency": + ret.MetricType = cloudprovider.BUCKET_METRIC_TYPE_LATECY + tags = map[string]string{"request": "get"} + case "get_request_count": + ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT + tags = map[string]string{"request": "get"} + case "request_count_4xx": + ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT + tags = map[string]string{"request": "4xx"} + case "request_count_5xx": + ret.MetricType = cloudprovider.BUCKET_METRYC_TYPE_REQ_COUNT + tags = map[string]string{"request": "5xx"} + default: + log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData[i].Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + +func (self *SHcsClient) getLoadbalancerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + params := map[string]interface{}{ + "from": fmt.Sprintf("%d", opts.StartTime.UnixMilli()), + "to": fmt.Sprintf("%d", opts.EndTime.UnixMilli()), + "period": "1", + "filter": "average", + } + metrics := []interface{}{} + namespace, dimesionName, metricNames := "SYS.ELB", "lb_instance_id", []string{ + "m7_in_Bps", + "m8_out_Bps", + "mc_l7_http_2xx", + "md_l7_http_3xx", + "me_l7_http_4xx", + "mf_l7_http_5xx", + } + + for _, metricName := range metricNames { + metrics = append(metrics, map[string]interface{}{ + "namespace": namespace, + "metric_name": metricName, + "dimensions": []map[string]string{ + { + "name": dimesionName, + "value": opts.ResourceId, + }, + }, + }) + } + params["metrics"] = metrics + resp, err := self.monitorPost("batch-query-metric-data", params) + if err != nil { + return nil, err + } + metricData := []MetricData{} + err = resp.Unmarshal(&metricData, "metrics") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + result := []cloudprovider.MetricValues{} + for i := range metricData { + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData[i].Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData[i].MetricName { + case "m7_in_Bps": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_RX + case "m8_out_Bps": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_TX + case "mc_l7_http_2xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "2xx"} + case "md_l7_http_3xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "3xx"} + case "md_l7_http_4xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "4xx"} + case "md_l7_http_5xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "5xx"} + default: + log.Warningf("invalid metricName %s for %s %s", metricData[i].MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData[i].Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + +func (self *SHcsClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + switch opts.ResourceType { + case cloudprovider.METRIC_RESOURCE_TYPE_SERVER: + return self.getServerMetrics(opts) + case cloudprovider.METRIC_RESOURCE_TYPE_REDIS: + return self.getRedisMetrics(opts) + case cloudprovider.METRIC_RESOURCE_TYPE_RDS: + return self.getRdsMetrics(opts) + case cloudprovider.METRIC_RESOURCE_TYPE_BUCKET: + return self.getBucketMetrics(opts) + case cloudprovider.METRIC_RESOURCE_TYPE_LB: + return self.getLoadbalancerMetrics(opts) + default: + return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "%s", opts.ResourceType) + } +} diff --git a/pkg/multicloud/hcs/nat_sku.go b/pkg/multicloud/hcs/nat_sku.go new file mode 100644 index 0000000000..cf40e235e5 --- /dev/null +++ b/pkg/multicloud/hcs/nat_sku.go @@ -0,0 +1,97 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +const ( + HUAWEI_NAT_SKU_SMALL = "Small" + HUAWEI_NAT_SKU_MIDDLE = "Middle" + HUAWEI_NAT_SKU_LARGE = "Large" + HUAWEI_NAT_SKU_XLARGE = "XLarge" +) + +type SNatSku struct { + spec string +} + +func (self *SNatSku) GetName() string { + return self.spec +} + +func (self *SNatSku) GetPps() int { + return 1000000 +} + +func (self *SNatSku) GetConns() int { + switch self.spec { + case HUAWEI_NAT_SKU_SMALL: + return 10000 + case HUAWEI_NAT_SKU_MIDDLE: + return 50000 + case HUAWEI_NAT_SKU_LARGE: + return 200000 + case HUAWEI_NAT_SKU_XLARGE: + return 1000000 + } + return 0 +} + +func (self *SNatSku) GetThroughput() int { + return 10 +} + +func (self *SNatSku) GetDesc() string { + switch self.spec { + case HUAWEI_NAT_SKU_SMALL: + return "小型" + case HUAWEI_NAT_SKU_MIDDLE: + return "中型" + case HUAWEI_NAT_SKU_LARGE: + return "大型" + case HUAWEI_NAT_SKU_XLARGE: + return "超大型" + } + return "" +} + +func (self *SNatSku) GetPostpaidStatus() string { + return api.NAT_SKU_AVAILABLE +} + +func (self *SNatSku) GetPrepaidStatus() string { + return api.NAT_SKU_SOLDOUT +} + +func (self *SNatSku) GetGlobalId() string { + return self.spec +} + +func (self *SRegion) GetICloudNatSkus() ([]cloudprovider.ICloudNatSku, error) { + ret := []cloudprovider.ICloudNatSku{} + for _, spec := range []string{ + HUAWEI_NAT_SKU_SMALL, + HUAWEI_NAT_SKU_MIDDLE, + HUAWEI_NAT_SKU_LARGE, + HUAWEI_NAT_SKU_XLARGE, + } { + sku := &SNatSku{spec: spec} + ret = append(ret, sku) + } + return ret, nil +} diff --git a/pkg/multicloud/hcs/natdtable.go b/pkg/multicloud/hcs/natdtable.go new file mode 100644 index 0000000000..44e4917ad8 --- /dev/null +++ b/pkg/multicloud/hcs/natdtable.go @@ -0,0 +1,149 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "net/url" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SNatDEntry struct { + multicloud.SResourceBase + multicloud.HcsTags + gateway *SNatGateway + + Id string `json:"id"` + NatGatewayId string `json:"nat_gateway_id"` + Protocol string `json:"protocol"` + Status string `json:"status"` + ExternalIP string `json:"floating_ip_address"` + ExternalPort int `json:"external_service_port"` + InternalIP string `json:"private_ip"` + InternalPort int `json:"internal_service_port"` + PortId string `json:"port_id"` + AdminStateUp bool `json:"admin_state_up"` +} + +func (nat *SNatDEntry) GetId() string { + return nat.Id +} + +func (nat *SNatDEntry) GetName() string { + // No name so return id + return nat.GetId() +} + +func (nat *SNatDEntry) GetGlobalId() string { + return nat.GetId() +} + +func (nat *SNatDEntry) GetStatus() string { + return NatResouceStatusTransfer(nat.Status) +} + +func (nat *SNatDEntry) GetIpProtocol() string { + return nat.Protocol +} + +func (nat *SNatDEntry) GetExternalIp() string { + return nat.ExternalIP +} + +func (nat *SNatDEntry) GetExternalPort() int { + return nat.ExternalPort +} + +func (nat *SNatDEntry) GetInternalIp() string { + return nat.InternalIP +} + +func (nat *SNatDEntry) GetInternalPort() int { + return nat.InternalPort +} + +func (nat *SNatDEntry) Delete() error { + return nat.gateway.region.DeleteNatDEntry(nat.GetId()) +} + +// getNatSTable return all snat rules of gateway +func (gateway *SNatGateway) getNatDTable() ([]SNatDEntry, error) { + ret, err := gateway.region.GetNatDTable(gateway.GetId()) + if err != nil { + return nil, err + } + for i := range ret { + ret[i].gateway = gateway + } + return ret, nil +} + +func (self *SRegion) GetNatDTable(natGatewayId string) ([]SNatDEntry, error) { + query := url.Values{} + query.Set("nat_gateway_id", natGatewayId) + ret := []SNatDEntry{} + err := self.list("nat", "v2.0", "dnat_rules", query, &ret) + if err != nil { + return nil, err + } + for i := range ret { + nat := &ret[i] + if len(nat.InternalIP) == 0 { + port, err := self.GetPort(nat.PortId) + if err != nil { + return nil, errors.Wrapf(err, `get port info for transfer to ip of port_id %q error`, nat.PortId) + } + nat.InternalIP = port.FixedIps[0].IpAddress + } + } + return ret, nil +} + +func (self *SRegion) DeleteNatDEntry(entryId string) error { + return self.delete("nat", "v2.0", "dnat_rules/"+entryId) +} + +func (nat *SNatDEntry) Refresh() error { + ret, err := nat.gateway.region.GetNatDEntry(nat.Id) + if err != nil { + return err + } + return jsonutils.Update(nat, ret) +} + +func (self *SRegion) CreateNatDEntry(rule cloudprovider.SNatDRule, gatewayId string) (*SNatDEntry, error) { + params := make(map[string]interface{}) + params["nat_gateway_id"] = gatewayId + params["private_ip"] = rule.InternalIP + params["internal_service_port"] = rule.InternalPort + params["floating_ip_id"] = rule.ExternalIPID + params["external_service_port"] = rule.ExternalPort + params["protocol"] = rule.Protocol + + packParams := map[string]interface{}{ + "dnat_rule": params, + } + ret := &SNatDEntry{} + return ret, self.create("nat", "v2.0", "dnat_rules", packParams, ret) +} + +func (self *SRegion) GetNatDEntry(id string) (*SNatDEntry, error) { + ret := &SNatDEntry{} + return ret, self.get("nat", "v2.0", "dnat_rules/"+id, ret) +} diff --git a/pkg/multicloud/hcs/natgateway.go b/pkg/multicloud/hcs/natgateway.go new file mode 100644 index 0000000000..3b384a42f9 --- /dev/null +++ b/pkg/multicloud/hcs/natgateway.go @@ -0,0 +1,265 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + billing_api "yunion.io/x/onecloud/pkg/apis/billing" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SNatGateway struct { + multicloud.SNatGatewayBase + multicloud.HcsTags + region *SRegion + + Id string + Name string + Description string + Spec string + Status string + InternalNetworkId string + CreatedTime string `json:"created_at"` +} + +func (gateway *SNatGateway) GetId() string { + return gateway.Id +} + +func (gateway *SNatGateway) GetName() string { + return gateway.Name +} + +func (gateway *SNatGateway) GetGlobalId() string { + return gateway.GetId() +} + +func (gateway *SNatGateway) GetStatus() string { + return NatResouceStatusTransfer(gateway.Status) +} + +func (self *SNatGateway) Delete() error { + return self.region.DeleteNatGateway(self.Id) +} + +func (self *SNatGateway) Refresh() error { + nat, err := self.region.GetNatGateway(self.Id) + if err != nil { + return errors.Wrapf(err, "GetNatGateway(%s)", self.Id) + } + return jsonutils.Update(self, nat) +} + +func (self *SNatGateway) GetINetworkId() string { + return self.InternalNetworkId +} + +func (gateway *SNatGateway) GetNatSpec() string { + switch gateway.Spec { + case "1": + return api.NAT_SPEC_SMALL + case "2": + return api.NAT_SPEC_MIDDLE + case "3": + return api.NAT_SPEC_LARGE + case "4": + return api.NAT_SPEC_XLARGE + } + return gateway.Spec +} + +func (gateway *SNatGateway) GetDescription() string { + return gateway.Description +} + +func (gateway *SNatGateway) GetBillingType() string { + return billing_api.BILLING_TYPE_POSTPAID +} + +func (gateway *SNatGateway) GetCreatedAt() time.Time { + t, _ := time.Parse("2006-01-02 15:04:05.000000", gateway.CreatedTime) + return t +} + +func (gateway *SNatGateway) GetExpiredAt() time.Time { + return time.Time{} +} + +func (gateway *SNatGateway) GetIEips() ([]cloudprovider.ICloudEIP, error) { + IEips, err := gateway.region.GetIEips() + if err != nil { + return nil, errors.Wrapf(err, `get all Eips of region %q error`, gateway.region.GetId()) + } + dNatTables, err := gateway.GetINatDTable() + if err != nil { + return nil, errors.Wrapf(err, `get all DNatTable of gateway %q error`, gateway.GetId()) + } + sNatTables, err := gateway.GetINatSTable() + if err != nil { + return nil, errors.Wrapf(err, `get all SNatTable of gateway %q error`, gateway.GetId()) + } + + // Get natIPSet of nat rules + natIPSet := make(map[string]struct{}) + for _, snat := range sNatTables { + natIPSet[snat.GetIP()] = struct{}{} + } + for _, dnat := range dNatTables { + natIPSet[dnat.GetExternalIp()] = struct{}{} + } + + // Add Eip whose GetIpAddr() in natIPSet to ret + ret := make([]cloudprovider.ICloudEIP, 0, 2) + for i := range IEips { + if _, ok := natIPSet[IEips[i].GetIpAddr()]; ok { + ret = append(ret, IEips[i]) + } + } + return ret, nil +} + +func (gateway *SNatGateway) GetINatDTable() ([]cloudprovider.ICloudNatDEntry, error) { + dNatTable, err := gateway.getNatDTable() + if err != nil { + return nil, errors.Wrapf(err, `get dnat table of nat gateway %q`, gateway.GetId()) + } + ret := make([]cloudprovider.ICloudNatDEntry, len(dNatTable)) + for i := range dNatTable { + ret[i] = &dNatTable[i] + } + return ret, nil +} + +func (gateway *SNatGateway) GetINatSTable() ([]cloudprovider.ICloudNatSEntry, error) { + sNatTable, err := gateway.getNatSTable() + if err != nil { + return nil, errors.Wrapf(err, `get dnat table of nat gateway %q`, gateway.GetId()) + } + ret := make([]cloudprovider.ICloudNatSEntry, len(sNatTable)) + for i := range sNatTable { + ret[i] = &sNatTable[i] + } + return ret, nil +} + +func (gateway *SNatGateway) CreateINatDEntry(rule cloudprovider.SNatDRule) (cloudprovider.ICloudNatDEntry, error) { + dnat, err := gateway.region.CreateNatDEntry(rule, gateway.GetId()) + if err != nil { + return nil, err + } + dnat.gateway = gateway + return dnat, nil +} + +func (gateway *SNatGateway) CreateINatSEntry(rule cloudprovider.SNatSRule) (cloudprovider.ICloudNatSEntry, error) { + snat, err := gateway.region.CreateNatSEntry(rule, gateway.GetId()) + if err != nil { + return nil, err + } + snat.gateway = gateway + return snat, nil +} + +func (gateway *SNatGateway) GetINatDEntryByID(id string) (cloudprovider.ICloudNatDEntry, error) { + dnat, err := gateway.region.GetNatDEntry(id) + if err != nil { + return nil, err + } + dnat.gateway = gateway + return dnat, nil +} + +func (gateway *SNatGateway) GetINatSEntryByID(id string) (cloudprovider.ICloudNatSEntry, error) { + snat, err := gateway.region.GetNatSEntry(id) + if err != nil { + return nil, err + } + snat.gateway = gateway + return snat, nil +} + +func (self *SRegion) GetNatGateways(vpcId string) ([]SNatGateway, error) { + query := url.Values{} + if len(vpcId) > 0 { + query.Set("router_id", vpcId) + } + ret := []SNatGateway{} + return ret, self.list("nat", "v2.0", "nat_gateways", query, &ret) +} + +func NatResouceStatusTransfer(status string) string { + // In Huawei Cloud, there are isx resource status of Nat, "ACTIVE", "PENDING_CREATE", + // "PENDING_UPDATE", "PENDING_DELETE", "EIP_FREEZED", "INACTIVE". + switch status { + case "ACTIVE": + return api.NAT_STAUTS_AVAILABLE + case "PENDING_CREATE": + return api.NAT_STATUS_ALLOCATE + case "PENDING_UPDATE", "PENDING_DELETE": + return api.NAT_STATUS_DEPLOYING + default: + return api.NAT_STATUS_UNKNOWN + } +} + +func (self *SRegion) GetNatGateway(id string) (*SNatGateway, error) { + ret := &SNatGateway{region: self} + return ret, self.get("nat", "v2.0", "nat_gateways/"+id, ret) +} + +func (self *SVpc) CreateINatGateway(opts *cloudprovider.NatGatewayCreateOptions) (cloudprovider.ICloudNatGateway, error) { + nat, err := self.region.CreateNatGateway(opts) + if err != nil { + return nil, errors.Wrapf(err, "CreateNatGateway") + } + return nat, nil +} + +func (self *SRegion) CreateNatGateway(opts *cloudprovider.NatGatewayCreateOptions) (*SNatGateway, error) { + spec := "" + switch strings.ToLower(opts.NatSpec) { + case api.NAT_SPEC_SMALL: + spec = "1" + case api.NAT_SPEC_MIDDLE: + spec = "2" + case api.NAT_SPEC_LARGE: + spec = "3" + case api.NAT_SPEC_XLARGE: + spec = "4" + } + params := map[string]interface{}{ + "nat_gateway": map[string]interface{}{ + "name": opts.Name, + "description": opts.Desc, + "router_id": opts.VpcId, + "internal_network_id": opts.NetworkId, + "spec": spec, + }, + } + ret := SNatGateway{region: self} + return &ret, self.create("nat", "v2.0", "nat_gateways", params, &ret) +} + +func (self *SRegion) DeleteNatGateway(id string) error { + return self.delete("nat", "v2.0", "nat_gateways/"+id) +} diff --git a/pkg/multicloud/hcs/natstable.go b/pkg/multicloud/hcs/natstable.go new file mode 100644 index 0000000000..52babbc98e --- /dev/null +++ b/pkg/multicloud/hcs/natstable.go @@ -0,0 +1,142 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "net/url" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SNatSEntry struct { + multicloud.SResourceBase + multicloud.HuaweiTags + gateway *SNatGateway + + Id string `json:"id"` + NatGatewayId string `json:"nat_gateway_id"` + NetworkId string `json:"network_id"` + SourceCIDR string `json:"cidr"` + Status string `json:"status"` + SNatIP string `json:"floating_ip_address"` + AdminStateUp bool `json:"admin_state_up"` +} + +func (nat *SNatSEntry) GetId() string { + return nat.Id +} + +func (nat *SNatSEntry) GetName() string { + // Snat rule has no name in Huawei Cloud, so return Id + return nat.GetId() +} + +func (nat *SNatSEntry) GetGlobalId() string { + return nat.GetId() +} + +func (nat *SNatSEntry) GetStatus() string { + return NatResouceStatusTransfer(nat.Status) +} + +func (nat *SNatSEntry) GetIP() string { + return nat.SNatIP +} + +func (nat *SNatSEntry) GetSourceCIDR() string { + return nat.SourceCIDR +} + +func (nat *SNatSEntry) GetNetworkId() string { + return nat.NetworkId +} + +func (nat *SNatSEntry) Delete() error { + return nat.gateway.region.DeleteNatSEntry(nat.GetId()) +} + +// getNatSTable return all snat rules of gateway +func (gateway *SNatGateway) getNatSTable() ([]SNatSEntry, error) { + ret, err := gateway.region.GetNatSTable(gateway.GetId()) + if err != nil { + return nil, err + } + for i := range ret { + ret[i].gateway = gateway + } + return ret, nil +} + +func (self *SRegion) GetNatSTable(natGatewayId string) ([]SNatSEntry, error) { + query := url.Values{} + query.Set("nat_gateway_id", natGatewayId) + ret := []SNatSEntry{} + err := self.list("nat", "v2.0", "snat_rules", query, &ret) + if err != nil { + return nil, err + } + for i := range ret { + nat := &ret[i] + if len(nat.SourceCIDR) != 0 { + continue + } + net, err := self.GetNetwork(nat.NetworkId) + if err != nil { + return nil, errors.Wrapf(err, "GetNetwork(%s)", nat.NetworkId) + } + nat.SourceCIDR = net.CIDR + } + return ret, nil +} + +func (self *SRegion) DeleteNatSEntry(entryId string) error { + return self.delete("nat", "v2.0", "snat_rules/"+entryId) +} + +func (nat *SNatSEntry) Refresh() error { + ret, err := nat.gateway.region.GetNatSEntry(nat.Id) + if err != nil { + return err + } + return jsonutils.Update(nat, ret) +} + +func (self *SRegion) CreateNatSEntry(rule cloudprovider.SNatSRule, gatewayId string) (*SNatSEntry, error) { + params := make(map[string]interface{}) + params["nat_gateway_id"] = gatewayId + if len(rule.NetworkID) != 0 { + params["network_id"] = rule.NetworkID + } + if len(rule.SourceCIDR) != 0 { + params["cidr"] = rule.SourceCIDR + } + params["floating_ip_id"] = rule.ExternalIPID + + packParams := map[string]interface{}{ + "snat_rule": params, + } + + ret := &SNatSEntry{} + return ret, self.create("nat", "v2.0", "snat_rules", packParams, ret) +} + +func (self *SRegion) GetNatSEntry(id string) (*SNatSEntry, error) { + ret := &SNatSEntry{} + return ret, self.get("nat", "v2.0", "snat_rules/"+id, ret) +} diff --git a/pkg/multicloud/hcs/network.go b/pkg/multicloud/hcs/network.go new file mode 100644 index 0000000000..e390923436 --- /dev/null +++ b/pkg/multicloud/hcs/network.go @@ -0,0 +1,167 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/netutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/rbacutils" +) + +/* +Subnets +*/ + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090590.html +type SNetwork struct { + multicloud.SResourceBase + multicloud.HcsTags + wire *SWire + + AvailabilityZone string `json:"availability_zone"` + CIDR string `json:"cidr"` + DHCPEnable bool `json:"dhcp_enable"` + DNSList []string `json:"dnsList"` + GatewayIP string `json:"gateway_ip"` + Id string `json:"id"` + Ipv6Enable bool `json:"ipv6_enable"` + Name string `json:"name"` + NeutronNetworkId string `json:"neutron_network_id"` + NeutronSubnetId string `json:"neutron_subnet_id"` + PrimaryDNS string `json:"primary_dns"` + SecondaryDNS string `json:"secondary_dns"` + Status string `json:"status"` + VpcId string `json:"vpc_id"` +} + +func (self *SNetwork) GetId() string { + return self.Id +} + +func (self *SNetwork) GetName() string { + if len(self.Name) == 0 { + return self.Id + } + return self.Name +} + +func (self *SNetwork) GetGlobalId() string { + return self.Id +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090591.html +func (self *SNetwork) GetStatus() string { + switch self.Status { + case "ACTIVE", "UNKNOWN": + return api.NETWORK_STATUS_AVAILABLE // ? todo: // UNKNOWN + case "ERROR": + return api.NETWORK_STATUS_UNKNOWN + default: + return api.NETWORK_STATUS_UNKNOWN + } +} + +func (self *SNetwork) Refresh() error { + ret, err := self.wire.region.GetNetwork(self.Id) + if err != nil { + return err + } + return jsonutils.Update(self, ret) +} + +func (self *SNetwork) GetIWire() cloudprovider.ICloudWire { + return self.wire +} + +func (self *SNetwork) GetIpStart() string { + pref, _ := netutils.NewIPV4Prefix(self.CIDR) + startIp := pref.Address.NetAddr(pref.MaskLen) // 0 + startIp = startIp.StepUp() // 1 + startIp = startIp.StepUp() // 2 + return startIp.String() +} + +func (self *SNetwork) GetIpEnd() string { + pref, _ := netutils.NewIPV4Prefix(self.CIDR) + endIp := pref.Address.BroadcastAddr(pref.MaskLen) // 255 + endIp = endIp.StepDown() // 254 + endIp = endIp.StepDown() // 253 + endIp = endIp.StepDown() // 252 + return endIp.String() +} + +func (self *SNetwork) GetIpMask() int8 { + pref, _ := netutils.NewIPV4Prefix(self.CIDR) + return pref.MaskLen +} + +func (self *SNetwork) GetGateway() string { + pref, _ := netutils.NewIPV4Prefix(self.CIDR) + startIp := pref.Address.NetAddr(pref.MaskLen) // 0 + startIp = startIp.StepUp() // 1 + return startIp.String() +} + +func (self *SNetwork) GetServerType() string { + return api.NETWORK_TYPE_GUEST +} + +func (self *SNetwork) GetIsPublic() bool { + return true +} + +func (self *SNetwork) GetPublicScope() rbacutils.TRbacScope { + return rbacutils.ScopeDomain +} + +func (self *SNetwork) Delete() error { + return self.wire.region.DeleteNetwork(self.Id) +} + +func (self *SNetwork) GetAllocTimeoutSeconds() int { + return 120 // 2 minutes +} + +func (self *SRegion) GetNetwork(id string) (*SNetwork, error) { + ret := &SNetwork{} + res := fmt.Sprintf("subnets/%s", id) + return ret, self.vpcGet(res, ret) +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090592.html +func (self *SRegion) GetNetwroks(vpcId string) ([]SNetwork, error) { + params := url.Values{} + if len(vpcId) > 0 { + params.Set("vpc_id", vpcId) + } + ret := []SNetwork{} + return ret, self.vpcList("subnets", params, &ret) +} + +func (self *SRegion) DeleteNetwork(id string) error { + res := fmt.Sprintf("subnets/%s", id) + return self.vpcDelete(res) +} + +func (self *SNetwork) GetProjectId() string { + return self.wire.vpc.EnterpriseProjectId +} diff --git a/pkg/multicloud/hcs/object.go b/pkg/multicloud/hcs/object.go new file mode 100644 index 0000000000..a58e83af02 --- /dev/null +++ b/pkg/multicloud/hcs/object.go @@ -0,0 +1,104 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "net/http" + + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.bucket +} + +func (o *SObject) GetAcl() cloudprovider.TBucketACLType { + acl := cloudprovider.ACLPrivate + obscli, err := o.bucket.region.getOBSClient() + if err != nil { + log.Errorf("o.bucket.region.GetOssClient error %s", err) + return acl + } + input := &obs.GetObjectAclInput{} + input.Bucket = o.bucket.Name + input.Key = o.Key + output, err := obscli.GetObjectAcl(input) + if err != nil { + log.Errorf("GetObjectAcl error: %v", err) + return acl + } + acl = obsAcl2CannedAcl(output.Grants) + return acl +} + +func (o *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error { + obscli, err := o.bucket.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "o.bucket.region.getOBSClient") + } + input := &obs.SetObjectAclInput{} + input.Bucket = o.bucket.Name + input.Key = o.Key + input.ACL = obs.AclType(string(aclStr)) + _, err = obscli.SetObjectAcl(input) + if err != nil { + return errors.Wrap(err, "obscli.SetObjectAcl") + } + return nil +} + +func (o *SObject) GetMeta() http.Header { + if o.Meta != nil { + return o.Meta + } + obscli, err := o.bucket.region.getOBSClient() + if err != nil { + log.Errorf("getOBSClient fail %s", err) + return nil + } + input := &obs.GetObjectMetadataInput{} + input.Bucket = o.bucket.Name + input.Key = o.Key + output, err := obscli.GetObjectMetadata(input) + if err != nil { + log.Errorf("obscli.GetObjectMetadata fail %s", err) + return nil + } + meta := http.Header{} + for k, v := range output.Metadata { + meta.Add(k, v) + } + if len(output.ContentType) > 0 { + meta.Add(cloudprovider.META_HEADER_CONTENT_TYPE, output.ContentType) + } + o.Meta = meta + return meta +} + +func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error { + return cloudprovider.ObjectSetMeta(ctx, o.bucket, o, meta) +} diff --git a/pkg/multicloud/hcs/port.go b/pkg/multicloud/hcs/port.go new file mode 100644 index 0000000000..dee0ea3c49 --- /dev/null +++ b/pkg/multicloud/hcs/port.go @@ -0,0 +1,161 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + "strings" + + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SFixedIP struct { + IpAddress string + SubnetId string + NetworkId string +} + +func (fixip *SFixedIP) GetGlobalId() string { + return fixip.IpAddress +} + +func (fixip *SFixedIP) GetIP() string { + return fixip.IpAddress +} + +func (fixip *SFixedIP) GetINetworkId() string { + return fixip.NetworkId +} + +func (fixip *SFixedIP) IsPrimary() bool { + return true +} + +type Port struct { + multicloud.SNetworkInterfaceBase + multicloud.HuaweiTags + region *SRegion + Id string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + AdminStateUp string `json:"admin_state_up"` + DNSName string `json:"dns_name"` + MACAddress string `json:"mac_address"` + NetworkId string `json:"network_id"` + TenantId string `json:"tenant_id"` + DeviceId string `json:"device_id"` + DeviceOwner string `json:"device_owner"` + BindingVnicType string `json:"binding:vnic_type"` + FixedIps []SFixedIP +} + +func (port *Port) GetName() string { + if len(port.Name) > 0 { + return port.Name + } + return port.Id +} + +func (port *Port) GetId() string { + return port.Id +} + +func (port *Port) GetGlobalId() string { + return port.Id +} + +func (port *Port) GetMacAddress() string { + return port.MACAddress +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0133195888.html +func (port *Port) GetAssociateType() string { + switch port.DeviceOwner { + case "compute:nova": + return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_SERVER + case "network:router_gateway", "network:router_interface", "network:router_interface_distributed": + return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_RESERVED + case "network:dhcp": + return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_DHCP + case "neutron:LOADBALANCERV2": + return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_LOADBALANCER + case "neutron:VIP_PORT": + return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_VIP + default: + if strings.HasPrefix(port.DeviceOwner, "compute:") { + return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_SERVER + } + } + return port.DeviceOwner +} + +func (port *Port) GetAssociateId() string { + return port.DeviceId +} + +func (port *Port) GetStatus() string { + switch port.Status { + case "ACTIVE", "DOWN": + return api.NETWORK_INTERFACE_STATUS_AVAILABLE + case "BUILD": + return api.NETWORK_INTERFACE_STATUS_CREATING + } + return port.Status +} + +func (port *Port) GetICloudInterfaceAddresses() ([]cloudprovider.ICloudInterfaceAddress, error) { + address := []cloudprovider.ICloudInterfaceAddress{} + for i := 0; i < len(port.FixedIps); i++ { + port.FixedIps[i].NetworkId = port.NetworkId + address = append(address, &port.FixedIps[i]) + } + return address, nil +} + +func (region *SRegion) GetINetworkInterfaces() ([]cloudprovider.ICloudNetworkInterface, error) { + ports, err := region.GetPorts("") + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudNetworkInterface{} + for i := 0; i < len(ports); i++ { + if len(ports[i].DeviceId) == 0 || !utils.IsInStringArray(ports[i].DeviceOwner, []string{"compute:CCI", "compute:nova", "neutron:LOADBALANCERV2"}) { + ports[i].region = region + ret = append(ret, &ports[i]) + } + } + return ret, nil +} + +func (self *SRegion) GetPort(id string) (*Port, error) { + port := &Port{} + res := fmt.Sprintf("ports/%s", id) + return port, self.vpcGet(res, port) +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0133195888.html +func (self *SRegion) GetPorts(instanceId string) ([]Port, error) { + ports := []Port{} + params := url.Values{} + if len(instanceId) > 0 { + params.Set("device_id", instanceId) + } + return ports, self.vpcList("ports", params, &ports) +} diff --git a/pkg/multicloud/hcs/project.go b/pkg/multicloud/hcs/project.go new file mode 100644 index 0000000000..53b03039bd --- /dev/null +++ b/pkg/multicloud/hcs/project.go @@ -0,0 +1,98 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "strings" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SProject struct { + Id string + Name string + DomainId string + Description bool + Enabled bool + ParentId string + IsDomain bool +} + +func (self *SProject) GetHealthStatus() string { + if self.Enabled { + return api.CLOUD_PROVIDER_HEALTH_NORMAL + } + return api.CLOUD_PROVIDER_HEALTH_SUSPENDED +} + +func (self *SHcsClient) GetProjects() ([]SProject, error) { + if len(self.projects) > 0 { + return self.projects, nil + } + resp, err := self.iamGet("v3/projects", nil) + if err != nil { + return nil, err + } + self.projects = []SProject{} + return self.projects, resp.Unmarshal(&self.projects, "projects") +} + +func (self *SHcsClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { + projects, err := self.GetProjects() + if err != nil { + return nil, err + } + + ret := make([]cloudprovider.SSubAccount, 0) + for i := range projects { + project := projects[i] + // name 为MOS的project是华为云内部的一个特殊project。不需要同步到本地 + if strings.ToLower(project.Name) == "mos" { + continue + } + s := cloudprovider.SSubAccount{ + Name: fmt.Sprintf("%s-%s", self.cpcfg.Name, project.Name), + Account: fmt.Sprintf("%s/%s", self.accessKey, project.Id), + HealthStatus: project.GetHealthStatus(), + DefaultProjectId: "0", + } + for j := range self.regions { + region := self.regions[j] + if strings.Contains(project.Name, region.Id) { + s.Desc = region.Locales.ZhCN + break + } + } + ret = append(ret, s) + } + return ret, nil +} + +func (self *SHcsClient) GetProjectById(id string) (*SProject, error) { + projects, err := self.GetProjects() + if err != nil { + return nil, err + } + for i := range projects { + if projects[i].Id == id { + return &projects[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, id) +} diff --git a/pkg/multicloud/hcs/provider/doc.go b/pkg/multicloud/hcs/provider/doc.go new file mode 100644 index 0000000000..dc40ffd062 --- /dev/null +++ b/pkg/multicloud/hcs/provider/doc.go @@ -0,0 +1 @@ +package provider // import "yunion.io/x/onecloud/pkg/multicloud/hcs/provider" diff --git a/pkg/multicloud/hcs/provider/provider.go b/pkg/multicloud/hcs/provider/provider.go new file mode 100644 index 0000000000..1ceff245aa --- /dev/null +++ b/pkg/multicloud/hcs/provider/provider.go @@ -0,0 +1,217 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package provider + +import ( + "context" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/multicloud/hcs" +) + +type SHcsProviderFactory struct { + cloudprovider.SPrivateCloudBaseProviderFactory +} + +func (self *SHcsProviderFactory) GetId() string { + return hcs.CLOUD_PROVIDER_HCS +} + +func (self *SHcsProviderFactory) GetName() string { + return hcs.CLOUD_PROVIDER_HCS_CN +} + +func (self *SHcsProviderFactory) IsCloudeventRegional() bool { + return true +} + +func (self *SHcsProviderFactory) GetMaxCloudEventSyncDays() int { + return 7 +} + +func (self *SHcsProviderFactory) GetMaxCloudEventKeepDays() int { + return 7 +} + +func (factory *SHcsProviderFactory) IsSupportModifyRouteTable() bool { + return true +} + +func (factory *SHcsProviderFactory) IsSupportSAMLAuth() bool { + return false +} + +func (self *SHcsProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) { + output := cloudprovider.SCloudaccount{} + if len(input.AccessKeyId) == 0 { + return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id") + } + if len(input.AccessKeySecret) == 0 { + return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret") + } + if len(input.AuthUrl) == 0 { + return output, errors.Wrap(httperrors.ErrMissingParameter, "auth_url") + } + output.Account = input.AccessKeyId + output.Secret = input.AccessKeySecret + output.AccessUrl = input.AuthUrl + return output, nil +} + +func (self *SHcsProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) { + output := cloudprovider.SCloudaccount{} + if len(input.AccessKeyId) == 0 { + return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id") + } + if len(input.AccessKeySecret) == 0 { + return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret") + } + output = cloudprovider.SCloudaccount{ + Account: input.AccessKeyId, + Secret: input.AccessKeySecret, + } + return output, nil +} + +func parseAccount(account string) (accessKey string, projectId string) { + segs := strings.Split(account, "/") + if len(segs) == 2 { + accessKey = segs[0] + projectId = segs[1] + } else { + accessKey = account + projectId = "" + } + return +} + +func (self *SHcsProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + accessKey, projectId := parseAccount(cfg.Account) + client, err := hcs.NewHcsClient( + hcs.NewHcsConfig( + accessKey, cfg.Secret, projectId, cfg.URL, + ).CloudproviderConfig(cfg), + ) + if err != nil { + return nil, err + } + return &SHcsProvider{ + SBaseProvider: cloudprovider.NewBaseProvider(self), + client: client, + }, nil +} + +func (self *SHcsProviderFactory) GetClientRC(info cloudprovider.SProviderInfo) (map[string]string, error) { + accessKey, projectId := parseAccount(info.Account) + return map[string]string{ + "HCS_AUTH_URL": info.Url, + "HCS_ACCESS_KEY": accessKey, + "HCS_SECRET": info.Secret, + "HCS_PROJECT_ID": projectId, + }, nil +} + +func init() { + factory := SHcsProviderFactory{} + cloudprovider.RegisterFactory(&factory) +} + +type SHcsProvider struct { + cloudprovider.SBaseProvider + client *hcs.SHcsClient +} + +func (self *SHcsProvider) GetVersion() string { + return "" +} + +func (self *SHcsProvider) GetSysInfo() (jsonutils.JSONObject, error) { + regions := self.client.GetIRegions() + info := jsonutils.NewDict() + info.Add(jsonutils.NewInt(int64(len(regions))), "region_count") + info.Add(jsonutils.NewString(hcs.HCS_API_VERSION), "api_version") + return info, nil +} + +func (self *SHcsProvider) GetIRegions() []cloudprovider.ICloudRegion { + return self.client.GetIRegions() +} + +func (self *SHcsProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) { + region, err := self.client.GetRegion(id) + if err != nil { + return nil, err + } + return region, nil +} + +func (self *SHcsProvider) GetBalance() (float64, string, error) { + return 0, api.CLOUD_PROVIDER_HEALTH_NORMAL, nil +} + +func (self *SHcsProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { + return self.client.GetSubAccounts() +} + +func (self *SHcsProvider) GetAccountId() string { + return self.client.GetAccountId() +} + +func (self *SHcsProvider) GetCloudRegionExternalIdPrefix() string { + return self.client.GetCloudRegionExternalIdPrefix() +} + +func (self *SHcsProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) { + return []cloudprovider.ICloudProject{}, nil +} + +func (self *SHcsProvider) CreateIProject(name string) (cloudprovider.ICloudProject, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (self *SHcsProvider) GetStorageClasses(regionId string) []string { + return []string{ + "STANDARD", "WARM", "COLD", + } +} + +func (self *SHcsProvider) GetBucketCannedAcls(regionId string) []string { + return []string{ + string(cloudprovider.ACLPrivate), + string(cloudprovider.ACLAuthRead), + string(cloudprovider.ACLPublicRead), + string(cloudprovider.ACLPublicReadWrite), + } +} + +func (self *SHcsProvider) GetObjectCannedAcls(regionId string) []string { + return []string{ + string(cloudprovider.ACLPrivate), + string(cloudprovider.ACLAuthRead), + string(cloudprovider.ACLPublicRead), + string(cloudprovider.ACLPublicReadWrite), + } +} + +func (self *SHcsProvider) GetCapabilities() []string { + return self.client.GetCapabilities() +} diff --git a/pkg/multicloud/hcs/redis.go b/pkg/multicloud/hcs/redis.go new file mode 100644 index 0000000000..a6022870b6 --- /dev/null +++ b/pkg/multicloud/hcs/redis.go @@ -0,0 +1,608 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + billing_api "yunion.io/x/onecloud/pkg/apis/billing" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/billing" +) + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423020.html +type SElasticcache struct { + multicloud.SElasticcacheBase + multicloud.HcsTags + + region *SRegion + + Name string `json:"name"` + Engine string `json:"engine"` + CapacityGB int `json:"capacity"` + IP string `json:"ip"` + DomainName string `json:"domainName"` + Port int `json:"port"` + Status string `json:"status"` + Libos bool `json:"libos"` + Description string `json:"description"` + Task string `json:"task"` + MaxMemoryMB int `json:"max_memory"` + UsedMemoryMB int `json:"used_memory"` + InstanceId string `json:"instance_id"` + ResourceSpecCode string `json:"resource_spec_code"` + EngineVersion string `json:"engine_version"` + InternalVersion string `json:"internal_version"` + ChargingMode int `json:"charging_mode"` + CapacityMinor string `json:"capacity_minor"` + VpcId string `json:"vpc_id"` + VpcName string `json:"vpc_name"` + TaskStatus string `json:"task_status"` + CreatedAt string `json:"created_at"` + ErrorCode string `json:"error_code"` + UserId string `json:"user_id"` + UserName string `json:"user_name"` + MaintainBegin string `json:"maintain_begin"` + MaintainEnd string `json:"maintain_end"` + NoPasswordAccess string `json:"no_password_access"` + AccessUser string `json:"access_user"` + EnablePublicip bool `json:"enable_publicip"` + PublicipId string `json:"publicip_id"` + PublicipAddress string `json:"publicip_address"` + EnableSSL bool `json:"enable_ssl"` + ServiceUpgrade bool `json:"service_upgrade"` + ServiceTaskId string `json:"service_task_id"` + IsFree string `json:"is_free"` + EnterpriseProjectId string `json:"enterprise_project_id"` + AvailableZones []string `json:"available_zones"` + SubnetId string `json:"subnet_id"` + SecurityGroupId string `json:"security_group_id"` + BackendAddrs []string `json:"backend_addrs"` + ProductId string `json:"product_id"` + SecurityGroupName string `json:"security_group_name"` + SubnetName string `json:"subnet_name"` + OrderId string `json:"order_id"` + SubnetCIdR string `json:"subnet_cidr"` + InstanceBackupPolicy string `json:"instance_backup_policy"` + EnterpriseProjectName string `json:"enterprise_project_name"` +} + +func (self *SElasticcache) GetId() string { + return self.InstanceId +} + +func (self *SElasticcache) GetName() string { + return self.Name +} + +func (self *SElasticcache) GetGlobalId() string { + return self.GetId() +} + +func (self *SElasticcache) GetProjectId() string { + return self.EnterpriseProjectId +} + +func (self *SElasticcache) Refresh() error { + cache, err := self.region.GetElasticCache(self.GetId()) + if err != nil { + return errors.Wrap(err, "Elasticcache.Refresh.GetElasticCache") + } + + err = jsonutils.Update(self, cache) + if err != nil { + return errors.Wrap(err, "Elasticcache.Refresh.Update") + } + + return nil +} + +func (self *SElasticcache) GetStatus() string { + switch self.Status { + case "RUNNING": + return api.ELASTIC_CACHE_STATUS_RUNNING + case "CREATING": + return api.ELASTIC_CACHE_STATUS_DEPLOYING + case "CREATEFAILED": + return api.ELASTIC_CACHE_STATUS_CREATE_FAILED + case "ERROR": + return api.ELASTIC_CACHE_STATUS_ERROR + case "RESTARTING": + return api.ELASTIC_CACHE_STATUS_RESTARTING + case "FROZEN": + return api.ELASTIC_CACHE_STATUS_UNAVAILABLE + case "EXTENDING": + return api.ELASTIC_CACHE_STATUS_CHANGING + case "RESTORING": + return api.ELASTIC_CACHE_STATUS_TRANSFORMING // ? + case "FLUSHING": + return api.ELASTIC_CACHE_STATUS_FLUSHING + } + + return "" +} + +func (self *SElasticcache) GetBillingType() string { + return billing_api.BILLING_TYPE_POSTPAID +} + +func (self *SElasticcache) GetCreatedAt() time.Time { + var createtime time.Time + if len(self.CreatedAt) > 0 { + createtime, _ = time.Parse("2006-01-02T15:04:05.000Z", self.CreatedAt) + } + + return createtime +} + +func (self *SElasticcache) GetInstanceType() string { + return self.ResourceSpecCode +} + +func (self *SElasticcache) GetCapacityMB() int { + return self.CapacityGB * 1024 +} + +func (self *SElasticcache) GetArchType() string { + if strings.Contains(self.ResourceSpecCode, "single") { + return api.ELASTIC_CACHE_ARCH_TYPE_SINGLE + } + if strings.Contains(self.ResourceSpecCode, "ha") { + return api.ELASTIC_CACHE_ARCH_TYPE_MASTER + } + if strings.Contains(self.ResourceSpecCode, "cluster") { + return api.ELASTIC_CACHE_ARCH_TYPE_CLUSTER + } + if strings.Contains(self.ResourceSpecCode, "proxy") { + return api.ELASTIC_CACHE_ARCH_TYPE_CLUSTER + } + return "" +} + +func (self *SElasticcache) GetNodeType() string { + // single（单副本） | double（双副本) + if strings.Contains(self.ResourceSpecCode, "single") { + return "single" + } + return "double" +} + +func (self *SElasticcache) GetEngine() string { + return self.Engine +} + +func (self *SElasticcache) GetEngineVersion() string { + return self.EngineVersion +} + +func (self *SElasticcache) GetVpcId() string { + return self.VpcId +} + +func (self *SElasticcache) GetZoneId() string { + if len(self.AvailableZones) > 0 { + zone, err := self.region.GetZoneById(self.AvailableZones[0]) + if err != nil { + log.Errorf("elasticcache.GetZoneId %s", err) + return "" + } + return zone.GetGlobalId() + } + return "" +} + +func (self *SElasticcache) GetNetworkType() string { + return api.LB_NETWORK_TYPE_VPC +} + +func (self *SElasticcache) GetNetworkId() string { + return self.SubnetId +} + +func (self *SElasticcache) GetPrivateDNS() string { + return self.DomainName +} + +func (self *SElasticcache) GetPrivateIpAddr() string { + return self.IP +} + +func (self *SElasticcache) GetPrivateConnectPort() int { + return self.Port +} + +func (self *SElasticcache) GetPublicDNS() string { + return self.PublicipAddress +} + +func (self *SElasticcache) GetPublicIpAddr() string { + return self.PublicipAddress +} + +func (self *SElasticcache) GetPublicConnectPort() int { + return self.Port +} + +func (self *SElasticcache) GetMaintainStartTime() string { + return self.MaintainBegin +} + +func (self *SElasticcache) GetMaintainEndTime() string { + return self.MaintainEnd +} + +func (self *SElasticcache) GetICloudElasticcacheAccounts() ([]cloudprovider.ICloudElasticcacheAccount, error) { + iaccounts := []cloudprovider.ICloudElasticcacheAccount{} + iaccount := &SElasticcacheAccount{cacheDB: self} + iaccounts = append(iaccounts, iaccount) + return iaccounts, nil +} + +func (self *SElasticcache) GetICloudElasticcacheAcls() ([]cloudprovider.ICloudElasticcacheAcl, error) { + // 华为云使用安全组做访问控制。目前未支持 + return []cloudprovider.ICloudElasticcacheAcl{}, nil +} + +func (self *SElasticcache) GetICloudElasticcacheBackups() ([]cloudprovider.ICloudElasticcacheBackup, error) { + start := self.GetCreatedAt().Format("20060102150405") + end := time.Now().Format("20060102150405") + backups, err := self.region.GetElasticCacheBackups(self.GetId(), start, end) + if err != nil { + return nil, err + } + + ibackups := make([]cloudprovider.ICloudElasticcacheBackup, len(backups)) + for i := range backups { + backups[i].cacheDB = self + ibackups[i] = &backups[i] + } + + return ibackups, nil +} + +func (self *SElasticcache) GetICloudElasticcacheParameters() ([]cloudprovider.ICloudElasticcacheParameter, error) { + parameters, err := self.region.GetElasticCacheParameters(self.GetId()) + if err != nil { + return nil, err + } + + iparameters := make([]cloudprovider.ICloudElasticcacheParameter, len(parameters)) + for i := range parameters { + parameters[i].cacheDB = self + iparameters[i] = ¶meters[i] + } + + return iparameters, nil +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423035.html +func (self *SRegion) GetElasticCacheBackups(instanceId, startTime, endTime string) ([]SElasticcacheBackup, error) { + params := url.Values{} + params.Set("beginTime", startTime) + params.Set("endTime", endTime) + res := fmt.Sprintf("instances/%s/backups", instanceId) + ret := []SElasticcacheBackup{} + return ret, self.redisList(res, params, &ret) +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423027.html +func (self *SRegion) GetElasticCacheParameters(instanceId string) ([]SElasticcacheParameter, error) { + params := url.Values{} + res := fmt.Sprintf("instances/%s/configs", instanceId) + ret := []SElasticcacheParameter{} + return ret, self.redisList(res, params, &ret) +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423044.html +func (self *SRegion) GetElasticCaches() ([]SElasticcache, error) { + params := url.Values{} + ret := []SElasticcache{} + return ret, self.redisList("instances", params, &ret) +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423020.html +func (self *SRegion) GetElasticCache(instanceId string) (*SElasticcache, error) { + cache := &SElasticcache{region: self} + res := fmt.Sprintf("instances/%s", instanceId) + return cache, self.redisGet(res, cache) +} + +func (self *SRegion) GetIElasticcacheById(id string) (cloudprovider.ICloudElasticcache, error) { + ec, err := self.GetElasticCache(id) + if err != nil { + return nil, errors.Wrapf(err, "GetElasticCache(%s)", id) + } + return ec, nil +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423047.html +func (self *SRegion) zoneNameToDcsZoneIds(zoneIds []string) ([]string, error) { + type Z struct { + Id string `json:"id"` + Code string `json:"code"` + Name string `json:"name"` + Port string `json:"port"` + ResourceAvailability string `json:"resource_availability"` + } + + rs := []Z{} + err := self.redisList("availableZones", url.Values{}, &rs) + if err != nil { + return nil, errors.Wrapf(err, "availableZones") + } + zoneMap := map[string]string{} + for i := range rs { + if rs[i].ResourceAvailability == "true" { + zoneMap[rs[i].Code] = rs[i].Id + } + } + + ret := []string{} + for _, zone := range zoneIds { + if id, ok := zoneMap[zone]; ok { + ret = append(ret, id) + } else { + return nil, errors.Wrap(fmt.Errorf("zone %s not found or not available", zone), "region.zoneNameToDcsZoneIds") + } + } + + return ret, nil +} + +func (self *SRegion) CreateElasticcache(opts *cloudprovider.SCloudElasticCacheInput) (*SElasticcache, error) { + params := map[string]interface{}{ + "name": opts.InstanceName, + "engine": opts.Engine, + "engine_version": opts.EngineVersion, + "capacity": opts.CapacityGB, + "vpc_id": opts.VpcId, + "subnet_id": opts.NetworkId, + "product_id": opts.InstanceType, + "spec_code": opts.InstanceType, + "available_zones": opts.ZoneIds, + "no_password_access": true, + "enable_publicip": false, + } + if len(opts.SecurityGroupIds) > 0 { + params["security_group_id"] = opts.SecurityGroupIds[0] + } + + if len(opts.ProjectId) > 0 { + params["enterprise_project_id"] = opts.ProjectId + } + + if len(opts.Password) > 0 { + params["no_password_access"] = false + params["password"] = opts.Password + if opts.Engine == "Memcache" { + params["access_user"] = opts.UserName + } + } + + if len(opts.EipId) > 0 { + params["enable_publicip"] = true + params["publicip_id"] = opts.EipId + } + + if len(opts.PrivateIpAddress) > 0 { + params["private_ip"] = opts.PrivateIpAddress + } + + if len(opts.MaintainBegin) > 0 { + params["maintain_begin"] = opts.MaintainBegin + params["maintain_end"] = opts.MaintainEnd + } + + ret := struct { + InstanceId string + }{} + err := self.redisCreate("instances", params, &ret) + if err != nil { + return nil, errors.Wrap(err, "redisCreate") + } + return self.GetElasticCache(ret.InstanceId) +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423019.html +func (self *SRegion) CreateIElasticcaches(opts *cloudprovider.SCloudElasticCacheInput) (cloudprovider.ICloudElasticcache, error) { + cache, err := self.CreateElasticcache(opts) + if err != nil { + return nil, err + } + return cache, nil +} + +func (self *SElasticcache) Restart() error { + return nil +} + +func (self *SElasticcache) Delete() error { + return self.region.DeleteElasticcache(self.InstanceId) +} + +func (self *SRegion) DeleteElasticcache(id string) error { + res := fmt.Sprintf("instances/%s", id) + return self.redisDelete(res) +} + +func (self *SElasticcache) ChangeInstanceSpec(spec string) error { + return cloudprovider.ErrNotImplemented +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423021.html +func (self *SElasticcache) SetMaintainTime(maintainStartTime, maintainEndTime string) error { + return cloudprovider.ErrNotImplemented +} + +// https://support.huaweicloud.com/usermanual-dcs/dcs-zh-ug-180314001.html +// 目前只有Redis3.0版本密码模式的实例支持通过公网访问Redis实例，其他版本暂不支持公网访问。 +// todo: 目前没找到api +func (self *SElasticcache) AllocatePublicConnection(port int) (string, error) { + return "", cloudprovider.ErrNotSupported +} + +// todo: 目前没找到api +func (self *SElasticcache) ReleasePublicConnection() error { + return cloudprovider.ErrNotSupported +} + +func (self *SElasticcache) CreateAccount(account cloudprovider.SCloudElasticCacheAccountInput) (cloudprovider.ICloudElasticcacheAccount, error) { + return nil, cloudprovider.ErrNotSupported +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423031.html + +func (self *SElasticcache) CreateAcl(aclName, securityIps string) (cloudprovider.ICloudElasticcacheAcl, error) { + return nil, cloudprovider.ErrNotSupported +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423029.html +func (self *SElasticcache) UpdateInstanceParameters(config jsonutils.JSONObject) error { + return cloudprovider.ErrNotImplemented +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423033.html +func (self *SElasticcache) CreateBackup(desc string) (cloudprovider.ICloudElasticcacheBackup, error) { + return nil, cloudprovider.ErrNotSupported +} + +func backupPeriodTrans(config cloudprovider.SCloudElasticCacheBackupPolicyUpdateInput) *jsonutils.JSONArray { + segs := strings.Split(config.PreferredBackupPeriod, ",") + ret := jsonutils.NewArray() + for _, seg := range segs { + switch seg { + case "Monday": + ret.Add(jsonutils.NewString("1")) + case "Tuesday": + ret.Add(jsonutils.NewString("2")) + case "Wednesday": + ret.Add(jsonutils.NewString("3")) + case "Thursday": + ret.Add(jsonutils.NewString("4")) + case "Friday": + ret.Add(jsonutils.NewString("5")) + case "Saturday": + ret.Add(jsonutils.NewString("6")) + case "Sunday": + ret.Add(jsonutils.NewString("7")) + } + } + + return ret +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423021.html +func (self *SElasticcache) UpdateBackupPolicy(config cloudprovider.SCloudElasticCacheBackupPolicyUpdateInput) error { + return nil +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423030.html +// 当前版本，只有DCS2.0实例支持清空数据功能，即flush操作。 +func (self *SElasticcache) FlushInstance(input cloudprovider.SCloudElasticCacheFlushInstanceInput) error { + return nil +} + +// SElasticcacheAccount => ResetPassword +func (self *SElasticcache) UpdateAuthMode(noPwdAccess bool, password string) error { + return cloudprovider.ErrNotSupported +} + +func (self *SElasticcache) GetAuthMode() string { + switch self.NoPasswordAccess { + case "true": + return "off" + default: + return "on" + } +} + +func (self *SElasticcache) GetSecurityGroupIds() ([]string, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (self *SElasticcache) GetICloudElasticcacheAccount(accountId string) (cloudprovider.ICloudElasticcacheAccount, error) { + accounts, err := self.GetICloudElasticcacheAccounts() + if err != nil { + return nil, errors.Wrap(err, "Elasticcache.GetICloudElasticcacheAccount.Accounts") + } + + for i := range accounts { + account := accounts[i] + if account.GetGlobalId() == accountId { + return account, nil + } + } + + return nil, cloudprovider.ErrNotFound +} + +func (self *SElasticcache) GetICloudElasticcacheAcl(aclId string) (cloudprovider.ICloudElasticcacheAcl, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (self *SElasticcache) GetICloudElasticcacheBackup(backupId string) (cloudprovider.ICloudElasticcacheBackup, error) { + backups, err := self.GetICloudElasticcacheBackups() + if err != nil { + return nil, err + } + + for _, backup := range backups { + if backup.GetId() == backupId { + return backup, nil + } + } + + return nil, cloudprovider.ErrNotFound +} + +func (instance *SElasticcache) SetTags(tags map[string]string, replace bool) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SElasticcache) UpdateSecurityGroups(secgroupIds []string) error { + return errors.Wrap(cloudprovider.ErrNotSupported, "UpdateSecurityGroups") +} + +func (self *SElasticcache) Renew(bc billing.SBillingCycle) error { + return cloudprovider.ErrNotSupported +} + +func (self *SElasticcache) SetAutoRenew(bc billing.SBillingCycle) error { + return cloudprovider.ErrNotSupported +} + +func (self *SRegion) GetIElasticcaches() ([]cloudprovider.ICloudElasticcache, error) { + caches, err := self.GetElasticCaches() + if err != nil { + return nil, err + } + + ret := []cloudprovider.ICloudElasticcache{} + for i := range caches { + caches[i].region = self + ret = append(ret, &caches[i]) + } + + return ret, nil +} diff --git a/pkg/multicloud/hcs/redis_account.go b/pkg/multicloud/hcs/redis_account.go new file mode 100644 index 0000000000..f5cc8a9032 --- /dev/null +++ b/pkg/multicloud/hcs/redis_account.go @@ -0,0 +1,83 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SElasticcacheAccount struct { + multicloud.SElasticcacheAccountBase + multicloud.HuaweiTags + + cacheDB *SElasticcache +} + +func (self *SElasticcacheAccount) GetId() string { + return fmt.Sprintf("%s/root", self.cacheDB.InstanceId) +} + +func (self *SElasticcacheAccount) GetName() string { + if len(self.cacheDB.AccessUser) > 0 { + return self.cacheDB.AccessUser + } + + return "root" +} + +func (self *SElasticcacheAccount) GetGlobalId() string { + return self.GetId() +} + +func (self *SElasticcacheAccount) GetStatus() string { + return api.ELASTIC_CACHE_ACCOUNT_STATUS_AVAILABLE +} + +func (self *SElasticcacheAccount) GetAccountType() string { + return api.ELASTIC_CACHE_ACCOUNT_TYPE_ADMIN +} + +func (self *SElasticcacheAccount) GetAccountPrivilege() string { + return api.ELASTIC_CACHE_ACCOUNT_PRIVILEGE_WRITE +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423031.html +// 未找到关闭密码的开放api， 不支持开启/关闭密码访问 +// https://console.huaweicloud.com/dcs/rest/v2/41f6bfe48d7f4455b7754f7c1b11ae34/instances/26db46e2-c7d8-4b5e-bd36-b5278d2fe17c/password/reset +// new_password: "26db46e2!" +// no_password_access: false +func (self *SElasticcacheAccount) ResetPassword(input cloudprovider.SCloudElasticCacheAccountResetPasswordInput) error { + return nil +} + +func (self *SElasticcacheAccount) UpdateAccount(input cloudprovider.SCloudElasticCacheAccountUpdateInput) error { + if input.Password != nil { + inputPassword := cloudprovider.SCloudElasticCacheAccountResetPasswordInput{} + inputPassword.NewPassword = *input.Password + inputPassword.OldPassword = input.OldPassword + inputPassword.NoPasswordAccess = input.NoPasswordAccess + return self.ResetPassword(inputPassword) + } + + return nil +} + +func (self *SElasticcacheAccount) Delete() error { + return cloudprovider.ErrNotSupported +} diff --git a/pkg/multicloud/hcs/redis_backup.go b/pkg/multicloud/hcs/redis_backup.go new file mode 100644 index 0000000000..d45c30eec3 --- /dev/null +++ b/pkg/multicloud/hcs/redis_backup.go @@ -0,0 +1,133 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423035.html +type SElasticcacheBackup struct { + multicloud.SElasticcacheBackupBase + multicloud.HuaweiTags + + cacheDB *SElasticcache + + Status string `json:"status"` + Remark string `json:"remark"` + Period string `json:"period"` + Progress string `json:"progress"` + SizeByte int64 `json:"size"` + InstanceId string `json:"instance_id"` + BackupId string `json:"backup_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ExecutionAt time.Time `json:"execution_at"` + BackupType string `json:"backup_type"` + BackupName string `json:"backup_name"` + ErrorCode string `json:"error_code"` + IsSupportRestore string `json:"is_support_restore"` +} + +func (self *SElasticcacheBackup) GetId() string { + return self.BackupId +} + +func (self *SElasticcacheBackup) GetName() string { + return self.BackupName +} + +func (self *SElasticcacheBackup) GetGlobalId() string { + return self.GetId() +} + +func (self *SElasticcacheBackup) Refresh() error { + cache, err := self.cacheDB.GetICloudElasticcacheBackup(self.GetId()) + if err != nil { + return errors.Wrap(err, "ElasticcacheBackup.Refresh.GetICloudElasticcacheBackup") + } + + err = jsonutils.Update(self, cache) + if err != nil { + return errors.Wrap(err, "ElasticcacheBackup.Refresh.Update") + } + + return nil +} + +func (self *SElasticcacheBackup) GetStatus() string { + switch self.Status { + case "waiting", "backuping": + return api.ELASTIC_CACHE_BACKUP_STATUS_CREATING + case "succeed": + return api.ELASTIC_CACHE_BACKUP_STATUS_SUCCESS + case "failed": + return api.ELASTIC_CACHE_BACKUP_STATUS_FAILED + case "expired": + return api.ELASTIC_CACHE_BACKUP_STATUS_CREATE_EXPIRED + case "deleted": + return api.ELASTIC_CACHE_BACKUP_STATUS_CREATE_DELETED + default: + return self.Status + } +} + +func (self *SElasticcacheBackup) GetBackupSizeMb() int { + return int(self.SizeByte / 1024 / 1024) +} + +func (self *SElasticcacheBackup) GetBackupType() string { + switch self.BackupType { + case "manual": + return api.ELASTIC_CACHE_BACKUP_MODE_MANUAL + case "auto": + return api.ELASTIC_CACHE_BACKUP_MODE_AUTOMATED + default: + return self.BackupType + } + +} + +func (self *SElasticcacheBackup) GetBackupMode() string { + return "" +} + +func (self *SElasticcacheBackup) GetDownloadURL() string { + return "" +} + +func (self *SElasticcacheBackup) GetStartTime() time.Time { + return self.CreatedAt +} + +func (self *SElasticcacheBackup) GetEndTime() time.Time { + return self.UpdatedAt +} + +func (self *SElasticcacheBackup) Delete() error { + return cloudprovider.ErrNotSupported +} + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423034.html +func (self *SElasticcacheBackup) RestoreInstance(instanceId string) error { + return nil +} diff --git a/pkg/multicloud/hcs/redis_instancetype.go b/pkg/multicloud/hcs/redis_instancetype.go new file mode 100644 index 0000000000..394d85bad2 --- /dev/null +++ b/pkg/multicloud/hcs/redis_instancetype.go @@ -0,0 +1,250 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SRedisSkuDetails struct { + Capacity int + MaxMemory float32 + MaxConnections int + MaxClients int + MaxBandwidth int + MaxInBandwidth int + TenantIpCount int + ShardingNum int + ProxyNum int + DBNumber int +} + +type SRedisFlavor struct { + Capacity int + Unit string + AvailableZones []string + AzCodes []string +} + +type SRedisInstanceType struct { + multicloud.SResourceBase + ProductId string + SpecCode string + CacheMode string + ProductType string + CpuType string + StorageType string + Details SRedisSkuDetails + Engine string + EngineVersions string + SpecDetails string + SpecDetails2 string + CharingType string + Price float32 + ProdType string + CloudServiceTypeCode string + CloudResourceTypeCode string + Flavors []SRedisFlavor +} + +type SRedisSku struct { + SRedisSkuDetails + SRedisFlavor + ProductId string + SpecCode string + CacheMode string + ProductType string + CpuType string + StorageType string + Engine string + EngineVersion string + ProdType string +} + +func (self *SRedisSku) GetName() string { + return self.SpecCode +} + +func (self *SRedisSku) GetGlobalId() string { + return fmt.Sprintf("%s-%s-%s-%s-%d-%.2f", self.SpecCode, self.Engine, self.EngineVersion, self.CacheMode, self.SRedisFlavor.Capacity, self.MaxMemory) +} + +func (self *SRedisSku) GetZoneId() string { + if len(self.AzCodes) > 0 { + return self.AzCodes[0] + } + return "" +} + +func (self *SRedisSku) GetSlaveZoneId() string { + return "" +} + +func (self *SRedisSku) GetEngineArch() string { + return self.CpuType +} + +func (self *SRedisSku) GetLocalCategory() string { + return self.CacheMode +} + +func (self *SRedisSku) GetPrepaidStatus() string { + return apis.SKU_STATUS_SOLDOUT +} + +func (self *SRedisSku) GetPostpaidStatus() string { + return apis.SKU_STATUS_AVAILABLE +} + +func (self *SRedisSku) GetEngine() string { + return self.Engine +} + +func (self *SRedisSku) GetEngineVersion() string { + return self.EngineVersion +} + +func (self *SRedisSku) GetCpuArch() string { + return self.CpuType +} + +func (self *SRedisSku) GetStorageType() string { + return self.StorageType +} + +func (self *SRedisSku) GetPerformanceType() string { + return "standard" +} + +func (self *SRedisSku) GetNodeType() string { + return self.CacheMode +} + +func (self *SRedisSku) GetDiskSizeGb() int { + return 0 +} + +func (self *SRedisSku) GetShardNum() int { + return self.ShardingNum +} + +func (self *SRedisSku) GetMaxShardNum() int { + return self.ShardingNum +} + +func (self *SRedisSku) GetReplicasNum() int { + return 0 +} + +func (self *SRedisSku) GetMaxReplicasNum() int { + return 0 +} + +func (self *SRedisSku) GetMaxClients() int { + return self.MaxClients +} + +func (self *SRedisSku) GetMaxConnections() int { + return self.MaxConnections +} + +func (self *SRedisSku) GetMaxInBandwidthMb() int { + return self.MaxInBandwidth +} + +func (self *SRedisSku) GetMemorySizeMb() int { + return self.SRedisSkuDetails.Capacity * 1024 +} + +func (self *SRedisSku) GetMaxMemoryMb() int { + return int(self.MaxMemory) * 1024 +} + +func (self *SRedisSku) GetQps() int { + return 0 +} + +func (self *SRegion) getRedisInstnaceTypes() ([]SRedisInstanceType, error) { + ret := []SRedisInstanceType{} + return ret, self.redisList("products", nil, &ret) +} + +func (self *SRegion) GetRedisInstnaceTypes() ([]SRedisSku, error) { + skus, err := self.getRedisInstnaceTypes() + if err != nil { + return nil, err + } + ret := []SRedisSku{} + for i := range skus { + details := []SRedisSkuDetails{} + detailsStr := strings.TrimPrefix(skus[i].SpecDetails2, "\n") + detailsStr = strings.TrimSuffix(detailsStr, "\n") + obj, err := jsonutils.ParseString(detailsStr) + if err != nil { + log.Errorf("parase details [%s], error: %v", detailsStr, err) + continue + } + err = obj.Unmarshal(&details) + if err != nil { + log.Errorf("unmarshal error: %v", err) + continue + } + for j := range details { + sku := SRedisSku{ + SRedisSkuDetails: details[j], + ProductId: skus[i].ProductId, + SpecCode: skus[i].SpecCode, + CacheMode: skus[i].CacheMode, + ProductType: skus[i].ProdType, + CpuType: skus[i].CpuType, + StorageType: skus[i].StorageType, + Engine: skus[i].Engine, + EngineVersion: skus[i].EngineVersions, + ProdType: skus[i].ProdType, + } + for k := range skus[i].Flavors { + if skus[i].Flavors[k].Capacity == details[j].Capacity { + sku.SRedisFlavor = skus[i].Flavors[k] + if len(sku.GetZoneId()) > 0 { + ret = append(ret, sku) + } + break + } + } + } + } + + return ret, nil +} + +func (self *SRegion) GetIElasticcacheSkus() ([]cloudprovider.ICloudElasticcacheSku, error) { + skus, err := self.GetRedisInstnaceTypes() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudElasticcacheSku{} + for i := range skus { + ret = append(ret, &skus[i]) + } + return ret, nil +} diff --git a/pkg/multicloud/hcs/redis_parameter.go b/pkg/multicloud/hcs/redis_parameter.go new file mode 100644 index 0000000000..a19009116f --- /dev/null +++ b/pkg/multicloud/hcs/redis_parameter.go @@ -0,0 +1,78 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/multicloud" +) + +// https://support.huaweicloud.com/api-dcs/dcs-zh-api-180423027.html +type SElasticcacheParameter struct { + multicloud.SElasticcacheParameterBase + multicloud.HuaweiTags + + cacheDB *SElasticcache + + Description string `json:"description"` + ParamId int64 `json:"param_id"` + ParamName string `json:"param_name"` + ParamValue string `json:"param_value"` + DefaultValue string `json:"default_value"` + ValueType string `json:"value_type"` + ValueRange string `json:"value_range"` +} + +func (self *SElasticcacheParameter) GetId() string { + return fmt.Sprintf("%d", self.ParamId) +} + +func (self *SElasticcacheParameter) GetName() string { + return self.ParamName +} + +func (self *SElasticcacheParameter) GetGlobalId() string { + return fmt.Sprintf("%s/%s", self.cacheDB.InstanceId, self.GetId()) +} + +func (self *SElasticcacheParameter) GetStatus() string { + return api.ELASTIC_CACHE_PARAMETER_STATUS_AVAILABLE +} + +func (self *SElasticcacheParameter) GetParameterKey() string { + return self.ParamName +} + +func (self *SElasticcacheParameter) GetParameterValue() string { + return self.ParamValue +} + +func (self *SElasticcacheParameter) GetParameterValueRange() string { + return self.Description +} + +func (self *SElasticcacheParameter) GetDescription() string { + return self.ValueRange +} + +func (self *SElasticcacheParameter) GetModifiable() bool { + return true +} + +func (self *SElasticcacheParameter) GetForceRestart() bool { + return false +} diff --git a/pkg/multicloud/hcs/region.go b/pkg/multicloud/hcs/region.go new file mode 100644 index 0000000000..7958bd3f65 --- /dev/null +++ b/pkg/multicloud/hcs/region.go @@ -0,0 +1,332 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + "strings" + + "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SRegion struct { + multicloud.HcsTags + multicloud.SRegion + multicloud.SRegionEipBase + multicloud.SRegionOssBase + multicloud.SRegionLbBase + multicloud.SRegionVpcBase + multicloud.SRegionZoneBase + multicloud.SRegionSecurityGroupBase + + obsClient *obs.ObsClient // 对象存储client.请勿直接引用。 + + client *SHcsClient + + Id string + CloudInfras []string + Description string + Locales struct { + EnUS string `json:"en-us"` + ZhCN string `json:"zh-cn"` + } + ParentRegionId string +} + +func (self *SRegion) GetClient() *SHcsClient { + return self.client +} + +func (self *SRegion) GetId() string { + return self.Id +} + +func (self *SRegion) GetGlobalId() string { + return fmt.Sprintf("%s/%s", CLOUD_PROVIDER_HCS, self.Id) +} + +func (self *SRegion) GetName() string { + return fmt.Sprintf("%s %s", CLOUD_PROVIDER_HCS_CN, self.Locales.ZhCN) +} + +func (self *SRegion) GetStatus() string { + return api.CLOUD_REGION_STATUS_INSERVER +} + +func (self *SRegion) GetILoadBalancerBackendGroups() ([]cloudprovider.ICloudLoadbalancerBackendGroup, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (self *SRegion) GetI18n() cloudprovider.SModelI18nTable { + en := fmt.Sprintf("%s %s", CLOUD_PROVIDER_HCS_EN, self.Locales.EnUS) + table := cloudprovider.SModelI18nTable{} + table["name"] = cloudprovider.NewSModelI18nEntry(self.GetName()).CN(self.GetName()).EN(en) + return table +} + +func (self *SRegion) GetCapabilities() []string { + return self.client.GetCapabilities() +} + +func (self *SRegion) GetCloudEnv() string { + return "" +} + +func (self *SRegion) GetProvider() string { + return CLOUD_PROVIDER_HCS +} + +func (self *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { + return cloudprovider.SGeographicInfo{} +} + +func (self *SRegion) ecsList(resource string, query url.Values, retVal interface{}) error { + return self.client.ecsList(self.Id, resource, query, retVal) +} + +func (self *SRegion) ecsGet(resource string, retVal interface{}) error { + return self.client.ecsGet(self.Id, resource, retVal) +} + +func (self *SRegion) ecsDelete(resource string) error { + return self.client.ecsDelete(self.Id, resource) +} + +func (self *SRegion) ecsCreate(resource string, body map[string]interface{}, retVal interface{}) error { + return self.client.ecsCreate(self.Id, resource, body, retVal) +} + +func (self *SRegion) ecsPerform(resource, action string, params map[string]interface{}, retVal interface{}) error { + return self.client.ecsPerform(self.Id, resource, action, params, retVal) +} + +func (region *SRegion) rdsList(resource string, query url.Values, retVal interface{}) error { + return region.client.rdsList(region.Id, resource, query, retVal) +} + +func (region *SRegion) rdsGet(resource string, retVal interface{}) error { + return region.client.rdsGet(region.Id, resource, retVal) +} + +func (region *SRegion) rdsDelete(resource string) error { + return region.client.rdsDelete(region.Id, resource) +} + +func (region *SRegion) rdsCreate(resource string, body map[string]interface{}, retVal interface{}) error { + return region.client.rdsCreate(region.Id, resource, body, retVal) +} + +func (self *SRegion) rdsPerform(resource, action string, params map[string]interface{}, retVal interface{}) error { + return self.client.rdsPerform(self.Id, resource, action, params, retVal) +} + +func (region *SRegion) rdsJobGet(resource string, query url.Values, retVal interface{}) error { + return region.client.rdsJobGet(region.Id, resource, query, retVal) +} + +func (region *SRegion) rdsDBPrivilegesDelete(resource string, params map[string]interface{}) error { + return region.client.rdsDBPrivvilegesDelete(region.Id, resource, params) +} + +func (region *SRegion) rdsDBPrivilegesGrant(resource string, params map[string]interface{}, retVal interface{}) error { + return region.client.rdsDBPrivilegesGrant(region.Id, resource, params, retVal) +} + +func (self *SRegion) perform(product, version, resource, action string, params map[string]interface{}, retVal interface{}) error { + return self.client.perform(product, version, self.Id, resource, action, params, retVal) +} + +func (self *SRegion) create(product, version, resource string, body map[string]interface{}, retVal interface{}) error { + return self.client.create(product, version, self.Id, resource, body, retVal) +} + +func (self *SRegion) delete(product, version, resource string) error { + return self.client.delete(product, version, self.Id, resource) +} + +func (self *SRegion) list(product, version, resource string, query url.Values, retVal interface{}) error { + return self.client.list(product, version, self.Id, resource, query, retVal) +} + +func (self *SRegion) get(product, version, resource string, retVal interface{}) error { + return self.client.get(product, version, self.Id, resource, retVal) +} +func (self *SRegion) update(product, version, resource string, params map[string]interface{}) error { + return self.client.update(product, version, self.Id, resource, params) +} + +func (self *SRegion) evsList(resource string, query url.Values, retVal interface{}) error { + return self.client.evsList(self.Id, resource, query, retVal) +} + +func (self *SRegion) evsGet(resource string, retVal interface{}) error { + return self.client.evsGet(self.Id, resource, retVal) +} + +func (self *SRegion) evsDelete(resource string) error { + return self.client.evsDelete(self.Id, resource) +} + +func (self *SRegion) evsPerform(resource, action string, params map[string]interface{}) error { + return self.client.evsPerform(self.Id, resource, action, params) +} + +func (self *SRegion) evsCreate(resource string, body map[string]interface{}, retVal interface{}) error { + return self.client.evsCreate(self.Id, resource, body, retVal) +} + +func (self *SRegion) vpcList(resource string, query url.Values, retVal interface{}) error { + return self.client.vpcList(self.Id, resource, query, retVal) +} + +func (self *SRegion) vpcCreate(resource string, body map[string]interface{}, retVal interface{}) error { + return self.client.vpcCreate(self.Id, resource, body, retVal) +} + +func (self *SRegion) vpcGet(resource string, retVal interface{}) error { + return self.client.vpcGet(self.Id, resource, retVal) +} + +func (self *SRegion) vpcUpdate(resource string, params map[string]interface{}) error { + return self.client.vpcUpdate(self.Id, resource, params) +} + +func (self *SRegion) vpcDelete(resource string) error { + return self.client.vpcDelete(self.Id, resource) +} + +func (self *SRegion) imsList(resource string, query url.Values, retVal interface{}) error { + return self.client.imsList(self.Id, resource, query, retVal) +} + +func (self *SRegion) imsCreate(resource string, body map[string]interface{}, retVal interface{}) error { + return self.client.imsCreate(self.Id, resource, body, retVal) +} + +func (self *SRegion) imsGet(resource string, retVal interface{}) error { + return self.client.imsGet(self.Id, resource, retVal) +} + +func (self *SRegion) imsUpdate(resource string, params map[string]interface{}) error { + return self.client.imsUpdate(self.Id, resource, params) +} + +func (self *SRegion) imsDelete(resource string) error { + return self.client.imsDelete(self.Id, resource) +} + +func (self *SRegion) imsPerform(resource, action string, params map[string]interface{}, retVal interface{}) error { + return self.client.imsPerform(self.Id, resource, action, params, retVal) +} + +func (self *SRegion) redisList(resource string, query url.Values, retVal interface{}) error { + return self.client.dcsList(self.Id, resource, query, retVal) +} + +func (self *SRegion) redisCreate(resource string, body map[string]interface{}, retVal interface{}) error { + return self.client.dcsCreate(self.Id, resource, body, retVal) +} + +func (self *SRegion) redisGet(resource string, retVal interface{}) error { + return self.client.dcsGet(self.Id, resource, retVal) +} + +func (self *SRegion) redisUpdate(resource string, params map[string]interface{}) error { + return self.client.dcsUpdate(self.Id, resource, params) +} + +func (self *SRegion) redisDelete(resource string) error { + return self.client.dcsDelete(self.Id, resource) +} + +func (self *SRegion) redisPerform(resource, action string, params map[string]interface{}, retVal interface{}) error { + return self.client.dcsPerform(self.Id, resource, action, params, retVal) +} + +func (self *SRegion) GetICloudSku(skuId string) (cloudprovider.ICloudSku, error) { + sku, err := self.GetchInstanceType(skuId) + if err != nil { + return nil, err + } + return sku, nil +} + +func (self *SRegion) getOBSClient() (*obs.ObsClient, error) { + if self.obsClient == nil { + obsClient, err := self.client.getOBSClient(self.GetId()) + if err != nil { + return nil, err + } + self.obsClient = obsClient + } + return self.obsClient, nil +} + +func (self *SRegion) HeadBucket(name string) (*obs.BaseModel, error) { + obsClient, err := self.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "region.getOBSClient") + } + return obsClient.HeadBucket(name) +} + +func (self *SRegion) getOBSEndpoint() string { + return self.client.getOBSEndpoint(self.GetId()) +} + +func str2StorageClass(storageClassStr string) (obs.StorageClassType, error) { + if strings.EqualFold(storageClassStr, string(obs.StorageClassStandard)) { + return obs.StorageClassStandard, nil + } else if strings.EqualFold(storageClassStr, string(obs.StorageClassWarm)) { + return obs.StorageClassWarm, nil + } else if strings.EqualFold(storageClassStr, string(obs.StorageClassCold)) { + return obs.StorageClassCold, nil + } else { + return obs.StorageClassStandard, errors.Error("unsupported storageClass") + } +} + +func (self *SRegion) GetBuckets() ([]SBucket, error) { + buckets, err := self.client.GetBuckets() + if err != nil { + return nil, err + } + ret := []SBucket{} + for i := range buckets { + if buckets[i].region.GetGlobalId() == self.GetGlobalId() { + ret = append(ret, buckets[i]) + } + } + return ret, nil +} + +func (self *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { + buckets, err := self.GetBuckets() + if err != nil { + return nil, errors.Wrap(err, "GetBuckets") + } + ret := make([]cloudprovider.ICloudBucket, 0) + for i := range buckets { + ret = append(ret, &buckets[i]) + } + return ret, nil +} diff --git a/pkg/multicloud/hcs/routetables.go b/pkg/multicloud/hcs/routetables.go new file mode 100644 index 0000000000..b7be1a328c --- /dev/null +++ b/pkg/multicloud/hcs/routetables.go @@ -0,0 +1,214 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +var hoptypes = map[string]string{ + api.NEXT_HOP_TYPE_VPCPEERING: "peering", + api.NEXT_HOP_TYPE_INSTANCE: "ecs", + api.NEXT_HOP_TYPE_NAT: "nat", + api.NEXT_HOP_TYPE_HAVIP: "vip", + api.NEXT_HOP_TYPE_NETWORK: "eni", + api.NEXT_HOP_TYPE_INTERVPCNETWORK: "cc", +} + +type SRoute struct { + multicloud.SResourceBase + multicloud.HuaweiTags + + Type string `json:"type"` + Destination string `json:"destination"` + Nexthop string `json:"nexthop"` + Description string `json:"description,omitempty"` +} + +func (self *SRoute) GetCidr() string { + return self.Destination +} + +func (self *SRoute) GetGlobalId() string { + return self.GetId() +} + +func (self *SRoute) GetId() string { + return fmt.Sprintf("%s:%s:%s", self.Type, self.Nexthop, self.Destination) +} + +func (self *SRoute) GetName() string { + return "" +} + +func (route *SRoute) GetStatus() string { + return api.ROUTE_ENTRY_STATUS_AVAILIABLE +} + +func (route *SRoute) GetType() string { + if route.Type != "local" { + return api.ROUTE_ENTRY_TYPE_CUSTOM + } + return api.ROUTE_ENTRY_TYPE_SYSTEM +} + +func (self *SRoute) GetNextHopType() string { + for k, v := range hoptypes { + if v == self.Type { + return k + } + } + return self.Type +} + +type Subnet struct { + Id string `json:"id"` +} + +type SRouteTable struct { + multicloud.SResourceBase + multicloud.HuaweiTags + vpc *SVpc + + Id string `json:"id"` + Name string `json:"name"` + Routes []SRoute `json:"routes"` + Subnets []Subnet `json:"subnets"` + VpcId string `json:"vpc_id"` + Default bool `json:"default"` + TenantId string `json:"tenant_id"` +} + +func (self *SRouteTable) GetDescription() string { + return "" +} + +func (self *SRouteTable) GetGlobalId() string { + return self.Id +} + +func (self *SRouteTable) GetId() string { + return self.Id +} + +func (self *SRouteTable) GetName() string { + return self.Name +} + +func (self *SRouteTable) GetRegionId() string { + return self.vpc.region.GetId() +} + +func (self *SRouteTable) GetVpcId() string { + return self.VpcId +} + +func (self *SRouteTable) GetType() cloudprovider.RouteTableType { + return cloudprovider.RouteTableTypeSystem +} + +func (self *SRouteTable) GetStatus() string { + return api.ROUTE_TABLE_AVAILABLE +} + +func (self *SRouteTable) Refresh() error { + rtb, err := self.vpc.region.GetRouteTable(self.Id) + if err != nil { + return err + } + return jsonutils.Update(self, rtb) +} + +func (self *SRouteTable) GetIRoutes() ([]cloudprovider.ICloudRoute, error) { + if len(self.Routes) == 0 { + err := self.Refresh() + if err != nil { + return nil, err + } + } + ret := []cloudprovider.ICloudRoute{} + for i := range self.Routes { + ret = append(ret, &self.Routes[i]) + } + return ret, nil +} + +func (self *SRoute) GetNextHop() string { + return self.Nexthop +} + +func (self *SRegion) GetRouteTables(vpcId string) ([]SRouteTable, error) { + params := url.Values{} + if len(vpcId) > 0 { + params.Set("vpc_id", vpcId) + } + rtbs := []SRouteTable{} + return rtbs, self.list("vpc", "v2.0", "vpc/routes", params, &rtbs) +} + +func (self *SRegion) GetRouteTable(id string) (*SRouteTable, error) { + tb := &SRouteTable{} + return tb, self.get("vpc", "v2.0", "vpc/routes/"+id, tb) +} + +func (self *SRouteTable) CreateRoute(route cloudprovider.RouteSet) error { + routeType, ok := hoptypes[route.NextHopType] + if !ok { + return errors.Wrapf(cloudprovider.ErrNotSupported, route.NextHopType) + } + params := map[string]interface{}{ + "routes": map[string]interface{}{ + "add": []map[string]interface{}{ + { + "type": routeType, + "destination": route.Destination, + "nexthop": route.NextHop, + }, + }, + }, + } + return self.vpc.region.create("vpc", "v2.0", "vpc/routes", params, nil) +} + +func (self *SRouteTable) RemoveRoute(route cloudprovider.RouteSet) error { + if len(route.RouteId) > 0 { + return self.vpc.region.delete("vpc", "v2.0", "vpc/routes/"+route.RouteId) + } + return fmt.Errorf("missing route id") +} + +func (self *SRouteTable) UpdateRoute(route cloudprovider.RouteSet) error { + err := self.RemoveRoute(route) + if err != nil { + return errors.Wrap(err, "self.RemoveRoute(route)") + } + err = self.CreateRoute(route) + if err != nil { + return errors.Wrap(err, "self.CreateRoute(route)") + } + return nil +} + +func (self *SRouteTable) GetAssociations() []cloudprovider.RouteTableAssociation { + return []cloudprovider.RouteTableAssociation{} +} diff --git a/pkg/multicloud/hcs/securitygroup.go b/pkg/multicloud/hcs/securitygroup.go new file mode 100644 index 0000000000..12d098ba8e --- /dev/null +++ b/pkg/multicloud/hcs/securitygroup.go @@ -0,0 +1,340 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net" + "net/url" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/secrules" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SecurityGroupRule struct { + Direction string `json:"direction"` + Ethertype string `json:"ethertype"` + Id string `json:"id"` + Description string `json:"description"` + SecurityGroupId string `json:"security_group_id"` + RemoteGroupId string `json:"remote_group_id"` +} + +type SecurityGroupRuleDetail struct { + Direction string `json:"direction"` + Ethertype string `json:"ethertype"` + Id string `json:"id"` + Description string `json:"description"` + PortRangeMax int64 `json:"port_range_max"` + PortRangeMin int64 `json:"port_range_min"` + Protocol string `json:"protocol"` + RemoteGroupId string `json:"remote_group_id"` + RemoteIPPrefix string `json:"remote_ip_prefix"` + SecurityGroupId string `json:"security_group_id"` + TenantId string `json:"tenant_id"` +} + +type SSecurityGroup struct { + multicloud.SSecurityGroup + multicloud.HcsTags + region *SRegion + + Id string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + VpcId string `json:"vpc_id"` + EnterpriseProjectId string `json:"enterprise_project_id "` + SecurityGroupRules []SecurityGroupRule `json:"security_group_rules"` +} + +// 判断是否兼容云端安全组规则 +func compatibleSecurityGroupRule(r SecurityGroupRule) bool { + // 忽略了源地址是安全组的规则 + if len(r.RemoteGroupId) > 0 { + return false + } + + // 忽略IPV6 + if r.Ethertype == "IPv6" { + return false + } + + return true +} + +func (self *SSecurityGroup) GetId() string { + return self.Id +} + +func (self *SSecurityGroup) GetVpcId() string { + return self.VpcId +} + +func (self *SSecurityGroup) GetName() string { + if len(self.Name) > 0 { + return self.Name + } + return self.Id +} + +func (self *SSecurityGroup) GetGlobalId() string { + return self.Id +} + +func (self *SSecurityGroup) GetStatus() string { + return "" +} + +func (self *SSecurityGroup) Refresh() error { + if new, err := self.region.GetSecurityGroupDetails(self.GetId()); err != nil { + return err + } else { + return jsonutils.Update(self, new) + } +} + +func (self *SSecurityGroup) IsEmulated() bool { + return false +} + +func (self *SSecurityGroup) GetDescription() string { + if self.Description == self.VpcId { + return "" + } + return self.Description +} + +// todo: 这里需要优化查询太多了 +func (self *SSecurityGroup) GetRules() ([]cloudprovider.SecurityRule, error) { + rules := make([]cloudprovider.SecurityRule, 0) + for _, r := range self.SecurityGroupRules { + if !compatibleSecurityGroupRule(r) { + continue + } + + rule, err := self.GetSecurityRule(r.Id) + if err != nil { + return rules, err + } + + rules = append(rules, rule) + } + + return rules, nil +} + +func (self *SSecurityGroup) GetSecurityRule(ruleId string) (cloudprovider.SecurityRule, error) { + remoteRule := SecurityGroupRuleDetail{} + err := self.region.vpcGet("security-group-rules/"+ruleId, &remoteRule) + if err != nil { + return cloudprovider.SecurityRule{}, err + } + + var direction secrules.TSecurityRuleDirection + if remoteRule.Direction == "ingress" { + direction = secrules.SecurityRuleIngress + } else { + direction = secrules.SecurityRuleEgress + } + + protocol := secrules.PROTO_ANY + if remoteRule.Protocol != "" { + protocol = remoteRule.Protocol + } + + var portStart int + var portEnd int + if protocol == secrules.PROTO_ICMP { + portStart = -1 + portEnd = -1 + } else { + portStart = int(remoteRule.PortRangeMin) + portEnd = int(remoteRule.PortRangeMax) + } + + ipNet := &net.IPNet{} + if len(remoteRule.RemoteIPPrefix) > 0 { + _, ipNet, err = net.ParseCIDR(remoteRule.RemoteIPPrefix) + } else { + _, ipNet, err = net.ParseCIDR("0.0.0.0/0") + } + + if err != nil { + return cloudprovider.SecurityRule{}, err + } + + rule := cloudprovider.SecurityRule{ + ExternalId: ruleId, + SecurityRule: secrules.SecurityRule{ + Priority: 1, + Action: secrules.SecurityRuleAllow, + IPNet: ipNet, + Protocol: protocol, + Direction: direction, + PortStart: portStart, + PortEnd: portEnd, + Ports: nil, + Description: remoteRule.Description, + }, + } + + err = rule.ValidateRule() + return rule, err +} + +func (self *SRegion) GetSecurityGroupDetails(id string) (*SSecurityGroup, error) { + ret := &SSecurityGroup{region: self} + return ret, self.vpcGet("security-groups/"+id, ret) +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090617.html +func (self *SRegion) GetSecurityGroups(vpcId string) ([]SSecurityGroup, error) { + query := url.Values{} + if len(vpcId) > 0 && !utils.IsInStringArray(vpcId, []string{"default", api.NORMAL_VPC_ID}) { // vpc_id = default or normal 时报错 '{"code":"VPC.0601","message":"Query security groups error vpcId is invalid."}' + query.Set("vpc_id", vpcId) + } + ret := []SSecurityGroup{} + return ret, self.vpcList("security-groups", query, &ret) +} + +func (self *SSecurityGroup) GetProjectId() string { + return "" +} + +func (self *SSecurityGroup) Delete() error { + return self.region.DeleteSecurityGroup(self.Id) +} + +func (self *SSecurityGroup) SyncRules(common, inAdds, outAdds, inDels, outDels []cloudprovider.SecurityRule) error { + for _, r := range append(inDels, outDels...) { + err := self.region.DeleteSecurityGroupRule(r.ExternalId) + if err != nil { + return errors.Wrapf(err, "delSecurityGroupRule(%s %s)", r.ExternalId, r.String()) + } + } + for _, r := range append(inAdds, outAdds...) { + err := self.region.CreateSecurityGroupRule(self.Id, r) + if err != nil { + return errors.Wrapf(err, "addSecurityGroupRule(%d %s)", r.Priority, r.String()) + } + } + return nil +} + +func (self *SRegion) DeleteSecurityGroup(id string) error { + return self.vpcDelete("security-groups/" + id) +} + +func (self *SRegion) DeleteSecurityGroupRule(ruleId string) error { + return self.vpcDelete("security-group-rules/" + ruleId) +} + +func (self *SRegion) CreateSecurityGroupRule(secGrpId string, rule cloudprovider.SecurityRule) error { + direction := "" + if rule.Direction == secrules.SecurityRuleIngress { + direction = "ingress" + } else { + direction = "egress" + } + + protocal := rule.Protocol + if rule.Protocol == secrules.PROTO_ANY { + protocal = "" + } + + // imcp协议默认为any + if rule.Protocol == secrules.PROTO_ICMP { + return self.AddSecurityGroupRule(secGrpId, direction, "-1", "-1", protocal, rule.IPNet.String()) + } + + if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + portStr := fmt.Sprintf("%d", port) + err := self.AddSecurityGroupRule(secGrpId, direction, portStr, portStr, protocal, rule.IPNet.String()) + if err != nil { + return err + } + } + } else { + portStart := fmt.Sprintf("%d", rule.PortStart) + portEnd := fmt.Sprintf("%d", rule.PortEnd) + err := self.AddSecurityGroupRule(secGrpId, direction, portStart, portEnd, protocal, rule.IPNet.String()) + if err != nil { + return err + } + } + return nil +} + +func (self *SRegion) AddSecurityGroupRule(secGrpId, direction, portStart, portEnd, protocol, ipNet string) error { + rule := map[string]interface{}{ + "security_group_id": secGrpId, + "direction": direction, + "remote_ip_prefix": ipNet, + "ethertype": "IPV4", + } + if len(portStart) > 0 && portStart != "0" && portStart != "-1" { + rule["port_range_min"] = portStart + } + if len(portEnd) > 0 && portEnd != "0" && portEnd != "-1" { + rule["port_range_max"] = portEnd + } + if len(protocol) > 0 { + rule["protocol"] = protocol + } + params := map[string]interface{}{ + "security_group_rule": rule, + } + return self.vpcCreate("security-group-rules", params, nil) +} + +func (self *SRegion) CreateSecurityGroup(vpcId string, name string, desc string) (*SSecurityGroup, error) { + params := map[string]interface{}{ + "security_group": map[string]interface{}{ + "name": name, + "vpc_id": vpcId, + }, + } + ret := &SSecurityGroup{region: self} + return ret, self.vpcCreate("security-groups", params, ret) +} + +func (self *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreateInput) (cloudprovider.ICloudSecurityGroup, error) { + return self.CreateSecurityGroup(conf.VpcId, conf.Name, conf.Desc) +} + +func (self *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICloudSecurityGroup, error) { + return self.GetSecurityGroupDetails(secgroupId) +} + +func (self *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, err := self.GetSecurityGroups(opts.VpcId) + if err != nil { + return nil, err + } + for i := range secgroups { + if secgroups[i].GetName() == opts.Name { + secgroups[i].region = self + return &secgroups[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "vpc:%s, name:%s", opts.VpcId, opts.Name) +} diff --git a/pkg/multicloud/hcs/shell/bucket.go b/pkg/multicloud/hcs/shell/bucket.go new file mode 100644 index 0000000000..b0db22ba42 --- /dev/null +++ b/pkg/multicloud/hcs/shell/bucket.go @@ -0,0 +1,43 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type BucketListOptions struct { + } + shellutils.R(&BucketListOptions{}, "bucket-list", "List buckets", func(cli *hcs.SRegion, args *BucketListOptions) error { + buckets, err := cli.GetBuckets() + if err != nil { + return err + } + printList(buckets, 0, 0, 0, nil) + return nil + }) + + type BucketCreateOptions struct { + NAME string + Storage string `default:"STANDARD" choices:"STANDARD|WARM|COLD"` + Acl string `default:"private" choices:"private|public-read|public-read-write"` + } + shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli *hcs.SRegion, args *BucketCreateOptions) error { + return cli.CreateBucket(args.NAME, args.Storage, args.Acl) + }) + +} diff --git a/pkg/multicloud/hcs/shell/dbinstance.go b/pkg/multicloud/hcs/shell/dbinstance.go new file mode 100644 index 0000000000..6e871dd6a9 --- /dev/null +++ b/pkg/multicloud/hcs/shell/dbinstance.go @@ -0,0 +1,97 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + huawei "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type DBInstanceListOptions struct { + } + shellutils.R(&DBInstanceListOptions{}, "dbinstance-list", "List dbinstances", func(cli *huawei.SRegion, args *DBInstanceListOptions) error { + dbinstances, err := cli.GetDBInstances() + if err != nil { + return err + } + printList(dbinstances, 0, 0, 0, nil) + return nil + }) + + type DBInstanceIdOptions struct { + ID string `help:"DBInstance ID"` + } + + shellutils.R(&DBInstanceIdOptions{}, "dbinstance-open-public-connection", "Open dbinstance public connection", func(cli *huawei.SRegion, args *DBInstanceIdOptions) error { + return cli.PublicConnectionAction(args.ID, "openRC") + }) + + shellutils.R(&DBInstanceIdOptions{}, "dbinstance-close-public-connection", "Close dbinstance public connection", func(cli *huawei.SRegion, args *DBInstanceIdOptions) error { + return cli.PublicConnectionAction(args.ID, "closeRC") + }) + + shellutils.R(&DBInstanceIdOptions{}, "dbinstance-show", "Show dbinstance", func(cli *huawei.SRegion, args *DBInstanceIdOptions) error { + dbinstance, err := cli.GetDBInstance(args.ID) + if err != nil { + return err + } + printObject(dbinstance) + return nil + }) + + shellutils.R(&DBInstanceIdOptions{}, "dbinstance-parameter-list", "Show dbinstance parameters", func(cli *huawei.SRegion, args *DBInstanceIdOptions) error { + parameters, err := cli.GetDBInstanceParameters(args.ID) + if err != nil { + return err + } + printList(parameters, 0, 0, 0, nil) + return nil + }) + + shellutils.R(&DBInstanceIdOptions{}, "dbinstance-backup-list", "Show dbinstance backups", func(cli *huawei.SRegion, args *DBInstanceIdOptions) error { + backups, err := cli.GetDBInstanceBackups(args.ID, "") + if err != nil { + return err + } + printList(backups, 0, 0, 0, nil) + return nil + }) + + type DBInstanceFlavorListOption struct { + ENGINE string `help:"DBInstance engine" choices:"MySQL|SQLServer|PostgreSQL"` + VERSION string `help:"DBInstance engine version" choices:"5.6|5.7|9.5|9.6|10.0|2014 SE|2014 EE|2016 SE|2016 EE|2008 R2 EE|2008 R2 WEB|2014 WEB|2016 WEB"` + } + + shellutils.R(&DBInstanceFlavorListOption{}, "dbinstance-flavor-list", "Show dbinstance backups", func(cli *huawei.SRegion, args *DBInstanceFlavorListOption) error { + flavors, err := cli.GetDBInstanceFlavors(args.ENGINE, args.VERSION) + if err != nil { + return err + } + printList(flavors, 0, 0, 0, nil) + return nil + }) + + type DBInstanceChangeConfigOptions struct { + INSTANCE string + InstanceType string + DiskSizeGB int + } + + shellutils.R(&DBInstanceChangeConfigOptions{}, "dbinstance-change-config", "Change dbinstance config", func(cli *huawei.SRegion, args *DBInstanceChangeConfigOptions) error { + return cli.ChangeDBInstanceConfig(args.INSTANCE, args.InstanceType, args.DiskSizeGB) + }) + +} diff --git a/pkg/multicloud/hcs/shell/dbinstance_account.go b/pkg/multicloud/hcs/shell/dbinstance_account.go new file mode 100644 index 0000000000..60375c0793 --- /dev/null +++ b/pkg/multicloud/hcs/shell/dbinstance_account.go @@ -0,0 +1,76 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + huawei "yunion.io/x/onecloud/pkg/multicloud/hcso" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type DBInstanceIdOptions struct { + ID string `help:"DBInstance ID"` + } + + shellutils.R(&DBInstanceIdOptions{}, "dbinstance-account-list", "Show dbinstance accounts", func(cli *huawei.SRegion, args *DBInstanceIdOptions) error { + accounts, err := cli.GetDBInstanceAccounts(args.ID) + if err != nil { + return err + } + printList(accounts, 0, 0, 0, nil) + return nil + }) + + type DBInstanceAccountDeleteOptions struct { + INSTANCE string + ACCOUNT string + } + + shellutils.R(&DBInstanceAccountDeleteOptions{}, "dbinstance-account-delete", "Delete dbinstance account", func(cli *huawei.SRegion, args *DBInstanceAccountDeleteOptions) error { + return cli.DeleteDBInstanceAccount(args.INSTANCE, args.ACCOUNT) + }) + + type DBInstanceAccountCreateOptions struct { + INSTANCE string + ACCOUNT string + PASSWORD string + } + + shellutils.R(&DBInstanceAccountCreateOptions{}, "dbinstance-account-create", "Create dbinstance account", func(cli *huawei.SRegion, args *DBInstanceAccountCreateOptions) error { + return cli.CreateDBInstanceAccount(args.INSTANCE, args.ACCOUNT, args.PASSWORD) + }) + + type DBInstanceAccountRevokePrivilegeOptions struct { + INSTANCE string + ACCOUNT string + DATABASE string + } + + shellutils.R(&DBInstanceAccountRevokePrivilegeOptions{}, "dbinstance-account-revoke-provilege", "Revoke dbinstance account privilege", func(cli *huawei.SRegion, args *DBInstanceAccountRevokePrivilegeOptions) error { + return cli.RevokeDBInstancePrivilege(args.INSTANCE, args.ACCOUNT, args.DATABASE) + }) + + type DBInstanceAccountGrantPrivilegeOptions struct { + INSTANCE string + ACCOUNT string + DATABASE string + PRIVILEGE string `help:"database privilege" choices:"r|rw"` + } + + shellutils.R(&DBInstanceAccountGrantPrivilegeOptions{}, "dbinstance-account-grant-provilege", "Grant dbinstance account privilege", func(cli *huawei.SRegion, args *DBInstanceAccountGrantPrivilegeOptions) error { + return cli.GrantDBInstancePrivilege(args.INSTANCE, args.ACCOUNT, args.DATABASE, args.PRIVILEGE) + }) + +} diff --git a/pkg/multicloud/hcs/shell/dbinstance_database.go b/pkg/multicloud/hcs/shell/dbinstance_database.go new file mode 100644 index 0000000000..afea978b93 --- /dev/null +++ b/pkg/multicloud/hcs/shell/dbinstance_database.go @@ -0,0 +1,55 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + huawei "yunion.io/x/onecloud/pkg/multicloud/hcso" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type DBInstanceIdOptions struct { + INSTANCE string `help:"DBInstance ID"` + } + + shellutils.R(&DBInstanceIdOptions{}, "dbinstance-database-list", "Show dbinstance databases", func(cli *huawei.SRegion, args *DBInstanceIdOptions) error { + databases, err := cli.GetDBInstanceDatabases(args.INSTANCE) + if err != nil { + return err + } + printList(databases, 0, 0, 0, nil) + return nil + }) + + type DBInstanceDatabaseDeleteOptions struct { + INSTANCE string + DATABASE string + } + + shellutils.R(&DBInstanceDatabaseDeleteOptions{}, "dbinstance-database-delete", "Delete dbinstance database", func(cli *huawei.SRegion, args *DBInstanceDatabaseDeleteOptions) error { + return cli.DeleteDBInstanceDatabase(args.INSTANCE, args.DATABASE) + }) + + type DBInstanceDatabaseCreateOptions struct { + INSTANCE string + DATABASE string + CHARACTER_SET string + } + + shellutils.R(&DBInstanceDatabaseCreateOptions{}, "dbinstance-database-create", "Create dbinstance database", func(cli *huawei.SRegion, args *DBInstanceDatabaseCreateOptions) error { + return cli.CreateDBInstanceDatabase(args.INSTANCE, args.DATABASE, args.CHARACTER_SET) + }) + +} diff --git a/pkg/multicloud/hcs/shell/dbinstance_skus.go b/pkg/multicloud/hcs/shell/dbinstance_skus.go new file mode 100644 index 0000000000..b95310dca4 --- /dev/null +++ b/pkg/multicloud/hcs/shell/dbinstance_skus.go @@ -0,0 +1,48 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + huawei "yunion.io/x/onecloud/pkg/multicloud/hcso" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type DBInstanceDatastoreOptions struct { + ENGINE string `help:"DBInstance Engine" choices:"MySQL|PostgreSQL|SQLServer"` + } + + shellutils.R(&DBInstanceDatastoreOptions{}, "dbinstance-datastore-list", "List dbinstance datastores", func(cli *huawei.SRegion, args *DBInstanceDatastoreOptions) error { + stores, err := cli.GetDBInstanceDatastores(args.ENGINE) + if err != nil { + return err + } + printList(stores, 0, 0, 0, nil) + return nil + }) + + type DBInstanceSkuListOptions struct { + } + + shellutils.R(&DBInstanceSkuListOptions{}, "dbinstance-sku-list", "List dbinstance datastores", func(cli *huawei.SRegion, args *DBInstanceSkuListOptions) error { + skus, err := cli.GetDBInstanceSkus() + if err != nil { + return err + } + printList(skus, 0, 0, 0, nil) + return nil + }) + +} diff --git a/pkg/multicloud/hcs/shell/disk.go b/pkg/multicloud/hcs/shell/disk.go new file mode 100644 index 0000000000..5b71cf6b9b --- /dev/null +++ b/pkg/multicloud/hcs/shell/disk.go @@ -0,0 +1,83 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type DiskListOptions struct { + Zone string `help:"Zone ID"` + } + shellutils.R(&DiskListOptions{}, "disk-list", "List disks", func(cli *hcs.SRegion, args *DiskListOptions) error { + disks, e := cli.GetDisks(args.Zone) + if e != nil { + return e + } + printList(disks, 0, 0, 0, nil) + return nil + }) + + type DiskCreateOptions struct { + ZONE string + STORAGE_TYPE string + NAME string + SIZE_GB int + SnapshotId string + Desc string + ProjectId string + } + + shellutils.R(&DiskCreateOptions{}, "disk-create", "Create disk", func(cli *hcs.SRegion, args *DiskCreateOptions) error { + disk, e := cli.CreateDisk(args.ZONE, args.STORAGE_TYPE, args.NAME, args.SIZE_GB, args.SnapshotId, args.Desc, args.ProjectId) + if e != nil { + return e + } + printObject(disk) + return nil + }) + + type DiskIdOptions struct { + ID string `help:"Disk ID"` + } + + shellutils.R(&DiskIdOptions{}, "disk-delete", "Delete disks", func(cli *hcs.SRegion, args *DiskIdOptions) error { + e := cli.DeleteDisk(args.ID) + if e != nil { + return e + } + return nil + }) + + shellutils.R(&DiskIdOptions{}, "disk-show", "Show disks", func(cli *hcs.SRegion, args *DiskIdOptions) error { + disk, e := cli.GetDisk(args.ID) + if e != nil { + return e + } + printObject(disk) + return nil + }) + + shellutils.R(&DiskListOptions{}, "disk-types", "List disk types", func(cli *hcs.SRegion, args *DiskListOptions) error { + ret, e := cli.GetDiskTypes() + if e != nil { + return e + } + printList(ret, 0, 0, 0, nil) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/doc.go b/pkg/multicloud/hcs/shell/doc.go new file mode 100644 index 0000000000..cdd19d8399 --- /dev/null +++ b/pkg/multicloud/hcs/shell/doc.go @@ -0,0 +1 @@ +package shell // import "yunion.io/x/onecloud/pkg/multicloud/hcs/shell" diff --git a/pkg/multicloud/hcs/shell/eip.go b/pkg/multicloud/hcs/shell/eip.go new file mode 100644 index 0000000000..41d1b12b42 --- /dev/null +++ b/pkg/multicloud/hcs/shell/eip.go @@ -0,0 +1,88 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type EipListOptions struct { + PortId string + Address []string + } + shellutils.R(&EipListOptions{}, "eip-list", "List eips", func(cli *hcs.SRegion, args *EipListOptions) error { + eips, err := cli.GetEips(args.PortId, args.Address) + if err != nil { + return nil + } + printList(eips, 0, 0, 0, nil) + return nil + }) + + type EipTypeListOptions struct { + } + + shellutils.R(&EipTypeListOptions{}, "eip-type-list", "List eip types", func(cli *hcs.SRegion, args *EipTypeListOptions) error { + types, err := cli.GetEipTypes() + if err != nil { + return nil + } + printList(types, 0, 0, 0, nil) + return nil + }) + + type EipAllocateOptions struct { + NAME string `help:"eip name"` + BW int `help:"Bandwidth limit in Mbps"` + BGP string `help:"bgp type"` + ChargeType string `help:"eip charge type" default:"traffic" choices:"traffic|bandwidth"` + ProjectId string + } + shellutils.R(&EipAllocateOptions{}, "eip-create", "Allocate an EIP", func(cli *hcs.SRegion, args *EipAllocateOptions) error { + eip, err := cli.AllocateEIP(args.NAME, args.BW, hcs.TInternetChargeType(args.ChargeType), args.BGP, args.ProjectId) + if err != nil { + return err + } + printObject(eip) + return nil + }) + + type EipReleaseOptions struct { + ID string `help:"EIP allocation ID"` + } + shellutils.R(&EipReleaseOptions{}, "eip-delete", "Release an EIP", func(cli *hcs.SRegion, args *EipReleaseOptions) error { + err := cli.DeallocateEIP(args.ID) + return err + }) + + type EipAssociateOptions struct { + ID string `help:"EIP allocation ID"` + INSTANCE string `help:"Instance ID"` + } + shellutils.R(&EipAssociateOptions{}, "eip-associate", "Associate an EIP", func(cli *hcs.SRegion, args *EipAssociateOptions) error { + return cli.AssociateEip(args.ID, args.INSTANCE) + }) + + type EipDissociateOptions struct { + ID string `help:"EIP allocation ID"` + } + + shellutils.R(&EipDissociateOptions{}, "eip-dissociate", "Dissociate an EIP", func(cli *hcs.SRegion, args *EipDissociateOptions) error { + return cli.DissociateEip(args.ID) + }) + +} diff --git a/pkg/multicloud/hcs/shell/image.go b/pkg/multicloud/hcs/shell/image.go new file mode 100644 index 0000000000..70a7023b2c --- /dev/null +++ b/pkg/multicloud/hcs/shell/image.go @@ -0,0 +1,56 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ImageListOptions struct { + Status string `help:"image status type" choices:"queued|saving|deleted|killed|active"` + Owner string `help:"Owner type" choices:"gold|private|shared"` + // Id []string `help:"Image ID"` + Name string `help:"image name"` + // Marker string `help:"marker"` + // Limit int `help:"page Limit"` + Env string `help:"virtualization env, e.g. FusionCompute, Ironic" choices:"FusionCompute|Ironic"` + } + shellutils.R(&ImageListOptions{}, "image-list", "List images", func(cli *hcs.SRegion, args *ImageListOptions) error { + images, e := cli.GetImages(args.Status, hcs.TImageOwnerType(args.Owner), args.Name, args.Env) + if e != nil { + return e + } + printList(images, 0, 0, 0, []string{}) + return nil + }) + + type ImageDeleteOptions struct { + ID string `help:"ID or Name to delete"` + } + shellutils.R(&ImageDeleteOptions{}, "image-delete", "Delete image", func(cli *hcs.SRegion, args *ImageDeleteOptions) error { + return cli.DeleteImage(args.ID) + }) + + shellutils.R(&ImageDeleteOptions{}, "image-show", "Show image", func(cli *hcs.SRegion, args *ImageDeleteOptions) error { + img, err := cli.GetImage(args.ID) + if err != nil { + return err + } + printObject(img) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/instance.go b/pkg/multicloud/hcs/shell/instance.go new file mode 100644 index 0000000000..d6dd34115d --- /dev/null +++ b/pkg/multicloud/hcs/shell/instance.go @@ -0,0 +1,261 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "context" + "fmt" + "strconv" + "strings" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type InstanceListOptions struct { + Ip string + } + shellutils.R(&InstanceListOptions{}, "instance-list", "List intances", func(cli *hcs.SRegion, args *InstanceListOptions) error { + instances, e := cli.GetInstances(args.Ip) + if e != nil { + return e + } + printList(instances, 0, 0, 0, nil) + return nil + }) + + type InstanceDiskOperationOptions struct { + ID string `help:"instance ID"` + DISK string `help:"disk ID"` + } + + type InstanceDiskAttachOptions struct { + ID string `help:"instance ID"` + DISK string `help:"disk ID"` + DEVICE string `help:"disk device name. eg. /dev/sdb"` + } + + shellutils.R(&InstanceDiskAttachOptions{}, "instance-attach-disk", "Attach a disk to instance", func(cli *hcs.SRegion, args *InstanceDiskAttachOptions) error { + err := cli.AttachDisk(args.ID, args.DISK, args.DEVICE) + if err != nil { + return err + } + return nil + }) + + shellutils.R(&InstanceDiskOperationOptions{}, "instance-detach-disk", "Detach a disk to instance", func(cli *hcs.SRegion, args *InstanceDiskOperationOptions) error { + err := cli.DetachDisk(args.ID, args.DISK) + if err != nil { + return err + } + return nil + }) + + type InstanceOperationOptions struct { + ID string `help:"instance ID"` + } + shellutils.R(&InstanceOperationOptions{}, "instance-start", "Start a instance", func(cli *hcs.SRegion, args *InstanceOperationOptions) error { + err := cli.StartVM(args.ID) + if err != nil { + return err + } + return nil + }) + + shellutils.R(&InstanceOperationOptions{}, "instance-interface-list", "List instance interface", func(cli *hcs.SRegion, args *InstanceOperationOptions) error { + ret, err := cli.GetInstanceInterfaces(args.ID) + if err != nil { + return err + } + printList(ret, 0, 0, 0, nil) + return nil + }) + + type InstanceStopOptions struct { + ID string `help:"instance ID"` + Force bool `help:"Force stop instance"` + } + shellutils.R(&InstanceStopOptions{}, "instance-stop", "Stop a instance", func(cli *hcs.SRegion, args *InstanceStopOptions) error { + err := cli.StopVM(args.ID, args.Force) + if err != nil { + return err + } + return nil + }) + shellutils.R(&InstanceOperationOptions{}, "instance-delete", "Delete a instance", func(cli *hcs.SRegion, args *InstanceOperationOptions) error { + err := cli.DeleteVM(args.ID) + if err != nil { + return err + } + return nil + }) + + /* + server-change-config 更改系统配置 + server-reset + */ + type InstanceDeployOptions struct { + ID string `help:"instance ID"` + Name string `help:"new instance name"` + Hostname string `help:"new hostname"` + Keypair string `help:"Keypair Name"` + DeleteKeypair bool `help:"Remove SSH keypair"` + Password string `help:"new password"` + // ResetPassword bool `help:"Force reset password"` + Description string `help:"new instances description"` + } + + shellutils.R(&InstanceDeployOptions{}, "instance-deploy", "Deploy keypair/password to a stopped virtual server", func(cli *hcs.SRegion, args *InstanceDeployOptions) error { + err := cli.DeployVM(args.ID, args.Name, args.Password, args.Keypair, args.DeleteKeypair, args.Description) + if err != nil { + return err + } + return nil + }) + + type InstanceRebuildRootOptions struct { + ID string `help:"instance ID"` + UserId string `help:"instance user ID"` + Image string `help:"Image ID"` + Password string `help:"admin password"` + PublicKeyName string `help:"public key name"` + UserData string `help:"cloud-init user data"` + } + + shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Reinstall virtual server system image", func(cli *hcs.SRegion, args *InstanceRebuildRootOptions) error { + ctx := context.Background() + return cli.ChangeRoot(ctx, args.UserId, args.ID, args.Image, args.Password, args.PublicKeyName, args.UserData) + }) + + type InstanceChangeConfigOptions struct { + ID string `help:"instance ID"` + InstanceType string `help:"instance type"` + } + + shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Deploy keypair/password to a stopped virtual server", func(cli *hcs.SRegion, args *InstanceChangeConfigOptions) error { + err := cli.ChangeVMConfig(args.ID, args.InstanceType) + if err != nil { + return err + } + return nil + }) + + type InstanceSaveImageOptions struct { + ID string `help:"Instance ID"` + IMAGE_NAME string `help:"Image name"` + Notes string `hlep:"Image desc"` + } + shellutils.R(&InstanceSaveImageOptions{}, "instance-save-image", "Save instance to image", func(cli *hcs.SRegion, args *InstanceSaveImageOptions) error { + opts := cloudprovider.SaveImageOptions{ + Name: args.IMAGE_NAME, + Notes: args.Notes, + } + image, err := cli.SaveImage(args.ID, &opts) + if err != nil { + return err + } + printObject(image) + return nil + }) + + type InstanceSetTagsOptions struct { + ID string `help:"Instance ID"` + Tags []string + } + shellutils.R(&InstanceSetTagsOptions{}, "instance-set-tags", "get intance metadata", func(cli *hcs.SRegion, args *InstanceSetTagsOptions) error { + tags := map[string]string{} + for i := range args.Tags { + splited := strings.Split(args.Tags[i], "=") + if len(splited) == 2 { + tags[splited[0]] = splited[1] + } + } + err := cli.CreateServerTags(args.ID, tags) + if err != nil { + return err + } + return nil + }) + + type InstanceDelTagsOptions struct { + ID string `help:"Instance ID"` + Tags []string + } + shellutils.R(&InstanceDelTagsOptions{}, "instance-del-tags", "del intance metadata", func(cli *hcs.SRegion, args *InstanceDelTagsOptions) error { + + err := cli.DeleteServerTags(args.ID, args.Tags) + if err != nil { + return err + } + return nil + }) + + type InstanceUpdateNameOptions struct { + ID string `help:"Instance ID"` + Name string + } + shellutils.R(&InstanceUpdateNameOptions{}, "instance-set-name", "set intance name", func(cli *hcs.SRegion, args *InstanceUpdateNameOptions) error { + + err := cli.UpdateVM(args.ID, args.Name) + if err != nil { + return err + } + return nil + }) + + type InstanceCreateOptions struct { + NAME string + IMAGE_ID string + INSTANCE_TYPE string + NETWORK_ID string + IpAddr string + Secgroup string `default:"default"` + VPC_ID string + ZONE_ID string + Desc string + Keypair string + Password string + UserData string + ProjectId string + DISKS []string + } + shellutils.R(&InstanceCreateOptions{}, "instance-create", "create instance", func(cli *hcs.SRegion, args *InstanceCreateOptions) error { + sysDisk := cloudprovider.SDiskInfo{} + dataDisks := []cloudprovider.SDiskInfo{} + for i, disk := range args.DISKS { + info := strings.Split(args.DISKS[i], ":") + if len(info) != 2 { + return fmt.Errorf("invalid disk %s", disk) + } + if i == 0 { + sysDisk.StorageType = info[0] + sysDisk.SizeGB, _ = strconv.Atoi(info[1]) + } else { + dataDisk := cloudprovider.SDiskInfo{} + dataDisk.StorageType = info[0] + dataDisk.SizeGB, _ = strconv.Atoi(info[1]) + dataDisks = append(dataDisks, dataDisk) + } + } + vm, err := cli.CreateInstance(args.NAME, args.IMAGE_ID, args.INSTANCE_TYPE, args.NETWORK_ID, args.Secgroup, args.VPC_ID, args.ZONE_ID, args.Desc, sysDisk, dataDisks, args.IpAddr, args.Keypair, args.Password, args.UserData, nil, args.ProjectId, nil) + if err != nil { + return err + } + printObject(vm) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/instancetype.go b/pkg/multicloud/hcs/shell/instancetype.go new file mode 100644 index 0000000000..61756de7fd --- /dev/null +++ b/pkg/multicloud/hcs/shell/instancetype.go @@ -0,0 +1,48 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type InstanceMatchOptions struct { + Zone string `help:"Test in zone"` + } + shellutils.R(&InstanceMatchOptions{}, "instance-type-list", "Select matching instance types", func(cli *hcs.SRegion, args *InstanceMatchOptions) error { + instanceTypes, e := cli.GetchInstanceTypes(args.Zone) + if e != nil { + return e + } + printList(instanceTypes, 0, 0, 0, []string{}) + return nil + }) + + type InstanceTypeIdOptions struct { + ID string + } + + shellutils.R(&InstanceTypeIdOptions{}, "instance-type-show", "show instance type", func(cli *hcs.SRegion, args *InstanceTypeIdOptions) error { + ret, e := cli.GetchInstanceType(args.ID) + if e != nil { + return e + } + printObject(ret) + return nil + }) + +} diff --git a/pkg/multicloud/hcs/shell/job.go b/pkg/multicloud/hcs/shell/job.go new file mode 100644 index 0000000000..4713d47e59 --- /dev/null +++ b/pkg/multicloud/hcs/shell/job.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type JobShowOptions struct { + ID string + Service string + } + shellutils.R(&JobShowOptions{}, "job-show", "show job", func(cli *hcs.SRegion, args *JobShowOptions) error { + job, err := cli.GetJob(args.Service, args.ID) + if err != nil { + return err + } + printObject(job) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/keypair.go b/pkg/multicloud/hcs/shell/keypair.go new file mode 100644 index 0000000000..07d5d7a785 --- /dev/null +++ b/pkg/multicloud/hcs/shell/keypair.go @@ -0,0 +1,46 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type KeyPairListOptions struct { + } + shellutils.R(&KeyPairListOptions{}, "keypair-list", "List keypairs", func(cli *hcs.SRegion, args *KeyPairListOptions) error { + keypairs, e := cli.GetKeypairs() + if e != nil { + return e + } + printList(keypairs, 0, 0, 0, []string{}) + return nil + }) + + type KeyPairImportOptions struct { + NAME string `help:"Name of new keypair"` + PUBKEY string `help:"Public key string"` + } + shellutils.R(&KeyPairImportOptions{}, "keypair-import", "Import a keypair", func(cli *hcs.SRegion, args *KeyPairImportOptions) error { + keypair, err := cli.ImportKeypair(args.NAME, args.PUBKEY) + if err != nil { + return err + } + printObject(keypair) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/lb.go b/pkg/multicloud/hcs/shell/lb.go new file mode 100644 index 0000000000..34e430199d --- /dev/null +++ b/pkg/multicloud/hcs/shell/lb.go @@ -0,0 +1,345 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ElbListOptions struct { + } + shellutils.R(&ElbListOptions{}, "elb-list", "List loadbalancers", func(cli *hcs.SRegion, args *ElbListOptions) error { + elbs, err := cli.GetILoadBalancers() + if err != nil { + return err + } + printList(elbs, len(elbs), 0, 0, []string{}) + return nil + }) + + type ElbCreateOptions struct { + Name string `help:"loadblancer name"` + SUBNET string `help:"subnet id"` + PrivateIP string `help:"loadbalancer private ip address"` + EipID string `help:"loadbalancer public ip id"` + } + shellutils.R(&ElbCreateOptions{}, "elb-create", "create loadbalancer", func(cli *hcs.SRegion, args *ElbCreateOptions) error { + loadbalancer := &cloudprovider.SLoadbalancer{ + Name: args.Name, + NetworkIDs: []string{args.SUBNET}, + EipID: args.EipID, + Address: args.PrivateIP, + } + + elb, err := cli.CreateLoadBalancer(loadbalancer) + if err != nil { + return err + } + + printObject(elb) + return nil + }) + + type ElbIdOptions struct { + ID string `help:"loadblancer id"` + } + shellutils.R(&ElbIdOptions{}, "elb-delete", "delete loadbalancer", func(cli *hcs.SRegion, args *ElbIdOptions) error { + err := cli.DeleteLoadBalancer(args.ID) + if err != nil { + return err + } + + return nil + }) + + shellutils.R(&ElbIdOptions{}, "elb-show", "show loadbalancer", func(cli *hcs.SRegion, args *ElbIdOptions) error { + lb, err := cli.GetLoadbalancer(args.ID) + if err != nil { + return err + } + printObject(lb) + return nil + }) + + type ElbListenerListOptions struct { + ElbID string `help:"loadblancer id"` + } + shellutils.R(&ElbListenerListOptions{}, "elb-listener-list", "list loadbalancer listener", func(cli *hcs.SRegion, args *ElbListenerListOptions) error { + listeners, err := cli.GetLoadBalancerListeners(args.ElbID) + if err != nil { + return err + } + + printList(listeners, len(listeners), 0, 0, []string{}) + return nil + }) + + type ElbListenerCreateOptions struct { + Name string `help:"listener name"` + Desc string `help:"listener Description"` + Http2 bool `help:"http2 enable status"` + PoolID string `help:"default backend group id"` + CertId string `help:"default certification id"` + XForwardedFor bool `help:"XForwardedFor enable status"` + LISTENER_TYPE string `help:"listener type" choices:"tcp|udp|http|https"` + LISTENER_PORT int `help:"listener port"` + ELB_ID string `help:"loadbalancer id"` + } + shellutils.R(&ElbListenerCreateOptions{}, "elb-listener-create", "create loadbalancer listener", func(cli *hcs.SRegion, args *ElbListenerCreateOptions) error { + input := &cloudprovider.SLoadbalancerListener{ + Name: args.Name, + LoadbalancerID: args.ELB_ID, + ListenerType: args.LISTENER_TYPE, + ListenerPort: args.LISTENER_PORT, + BackendGroupID: args.PoolID, + EnableHTTP2: args.Http2, + CertificateID: args.CertId, + Description: args.Desc, + XForwardedFor: args.XForwardedFor, + } + listener, err := cli.CreateLoadBalancerListener(input) + if err != nil { + return err + } + + printObject(listener) + return nil + }) + + type ElbListenerUpdateOptions struct { + Name string `help:"listener name"` + Desc string `help:"listener Description"` + Http2 bool `help:"http2 enable status"` + PoolID string `help:"default backend group id"` + CertId string `help:"default certification id"` + XForwardedFor bool `help:"XForwardedFor enable status"` + LISTENER_ID string `help:"listener id"` + } + shellutils.R(&ElbListenerUpdateOptions{}, "elb-listener-update", "update loadbalancer listener", func(cli *hcs.SRegion, args *ElbListenerUpdateOptions) error { + input := &cloudprovider.SLoadbalancerListener{ + Name: args.Name, + BackendGroupID: args.PoolID, + EnableHTTP2: args.Http2, + CertificateID: args.CertId, + Description: args.Desc, + XForwardedFor: args.XForwardedFor, + } + + err := cli.UpdateLoadBalancerListener(args.LISTENER_ID, input) + if err != nil { + return err + } + + return nil + }) + + type ElbBackendGroupListOptions struct { + ElbID string `help:"loadbalancer id"` + } + shellutils.R(&ElbBackendGroupListOptions{}, "elb-backend-group-list", "List backend groups", func(cli *hcs.SRegion, args *ElbBackendGroupListOptions) error { + elbbg, err := cli.GetLoadBalancerBackendGroups(args.ElbID) + if err != nil { + return err + } + + printList(elbbg, len(elbbg), 0, 0, []string{}) + return nil + }) + + type ElbBackendGroupCreateOptions struct { + Name string `help:"backend group name"` + Desc string `help:"backend group description"` + PROTOCOL string `help:"backend group protocol" choices:"tcp|udp|http"` + ALGORITHM string `help:"backend group algorithm" choices:"rr|wlc|sch"` + ListenerID string `help:"listener id to binding"` + ElbID string `help:"loadbalancer id belong to"` + StickySessionType string `help:"sticky session type" choices:"insert|server"` + StickySessionCookieName string `help:"sticky session cookie name"` + StickySessionTimeout int `help:"sticky session timeout. udp/tcp 1~60. http 1~1440"` + HealthCheck bool `help:"enable health check"` + HealthCheckType string `help:"health check type protocol" choices:"tcp|udp|http"` + HealthCheckTimeout int `help:"health check timeout"` + HealthCheckDomain string `help:"health check domain"` + HealthCheckURI string `help:"health check uri path"` + HealthCheckInterval int `help:"health check interval"` + HealthCheckRise int `help:"health check max retries"` + } + shellutils.R(&ElbBackendGroupCreateOptions{}, "elb-backend-group-create", "Create backend groups", func(cli *hcs.SRegion, args *ElbBackendGroupCreateOptions) error { + var health *cloudprovider.SLoadbalancerHealthCheck + if args.HealthCheck { + health = &cloudprovider.SLoadbalancerHealthCheck{ + HealthCheckType: args.HealthCheckType, + HealthCheckTimeout: args.HealthCheckTimeout, + HealthCheckDomain: args.HealthCheckDomain, + HealthCheckURI: args.HealthCheckURI, + HealthCheckInterval: args.HealthCheckInterval, + HealthCheckRise: args.HealthCheckRise, + } + } + + var sticky *cloudprovider.SLoadbalancerStickySession + if len(args.StickySessionType) > 0 { + sticky = &cloudprovider.SLoadbalancerStickySession{ + StickySessionCookie: args.StickySessionCookieName, + StickySessionType: args.StickySessionType, + StickySessionCookieTimeout: args.StickySessionTimeout, + } + } + + group := &cloudprovider.SLoadbalancerBackendGroup{ + Name: args.Name, + LoadbalancerID: args.ElbID, + ListenerID: args.ListenerID, + ListenType: args.PROTOCOL, + Scheduler: args.ALGORITHM, + StickySession: sticky, + HealthCheck: health, + } + + elbbg, err := cli.CreateLoadBalancerBackendGroup(group) + if err != nil { + return err + } + + printObject(elbbg) + return nil + }) + + type ElbBackendGroupUpdateOptions struct { + POOL_ID string `help:"backend group id"` + Name string `help:"backend group name"` + } + shellutils.R(&ElbBackendGroupUpdateOptions{}, "elb-backend-group-update", "Update backend groups", func(cli *hcs.SRegion, args *ElbBackendGroupUpdateOptions) error { + group := &cloudprovider.SLoadbalancerBackendGroup{ + Name: args.Name, + } + + elbbg, err := cli.UpdateLoadBalancerBackendGroup(args.POOL_ID, group) + if err != nil { + return err + } + + printObject(elbbg) + return nil + }) + + type ElbBackendGroupDeleteOptions struct { + POOL_ID string `help:"backend group id"` + } + shellutils.R(&ElbBackendGroupDeleteOptions{}, "elb-backend-group-delete", "Delete backend group", func(cli *hcs.SRegion, args *ElbBackendGroupDeleteOptions) error { + err := cli.DeleteLoadBalancerBackendGroup(args.POOL_ID) + if err != nil { + return err + } + + return nil + }) + + type ElbBackendAddOptions struct { + Name string `help:"backend name"` + POOL_ID string `help:"backend group id"` + SUBNET_ID string `help:"instance subnet id"` + ADDRESS string `help:"instance ip address"` + PORT int `help:"backend protocol port [1，65535]"` + Weight int `help:"backend weight [0，100]" default:"1"` + } + shellutils.R(&ElbBackendAddOptions{}, "elb-backend-add", "Add backend to backendgroup", func(cli *hcs.SRegion, args *ElbBackendAddOptions) error { + elbb, err := cli.AddLoadBalancerBackend(args.POOL_ID, args.SUBNET_ID, args.ADDRESS, args.PORT, args.Weight) + if err != nil { + return err + } + + printObject(elbb) + return nil + }) + + type ElbBackendListOptions struct { + POOL_ID string `help:"backend group id"` + } + shellutils.R(&ElbBackendListOptions{}, "elb-backend-list", "list backend", func(cli *hcs.SRegion, args *ElbBackendListOptions) error { + elbb, err := cli.GetLoadBalancerBackends(args.POOL_ID) + if err != nil { + return err + } + + printList(elbb, len(elbb), 0, 0, []string{}) + return nil + }) + + type ElbListenerPolicyListOptions struct { + ListenerID string `help:"listener id"` + } + shellutils.R(&ElbListenerPolicyListOptions{}, "elb-listener-policy-list", "List listener policies", func(cli *hcs.SRegion, args *ElbListenerPolicyListOptions) error { + elblp, err := cli.GetLoadBalancerPolicies(args.ListenerID) + if err != nil { + return err + } + + printList(elblp, len(elblp), 0, 0, []string{}) + return nil + }) + + type ElbListenerPolicyCreateOptions struct { + LISTENER_ID string `help:"listener id"` + Name string `help:"policy name"` + Domain string `help:"policy domain"` + Path string `help:"policy path"` + PoolID string `help:"backend group name"` + } + shellutils.R(&ElbListenerPolicyCreateOptions{}, "elb-listener-policy-create", "Create listener policy", func(cli *hcs.SRegion, args *ElbListenerPolicyCreateOptions) error { + rule := &cloudprovider.SLoadbalancerListenerRule{ + Name: args.Name, + Domain: args.Domain, + Path: args.Path, + BackendGroupID: args.PoolID, + } + + elblp, err := cli.CreateLoadBalancerPolicy(args.LISTENER_ID, rule) + if err != nil { + return err + } + + printObject(elblp) + return nil + }) + + type ElbListenerPolicyDeleteOptions struct { + POLICY_ID string `help:"policy id"` + } + shellutils.R(&ElbListenerPolicyDeleteOptions{}, "elb-listener-policy-delete", "Delete listener policy", func(cli *hcs.SRegion, args *ElbListenerPolicyDeleteOptions) error { + err := cli.DeleteLoadBalancerPolicy(args.POLICY_ID) + if err != nil { + return err + } + + return nil + }) + + type ElbListenerPolicyRuleListOptions struct { + POLICY_ID string `help:"policy id"` + } + shellutils.R(&ElbListenerPolicyRuleListOptions{}, "elb-listener-policyrule-list", "List listener policy rules", func(cli *hcs.SRegion, args *ElbListenerPolicyRuleListOptions) error { + elblpr, err := cli.GetLoadBalancerPolicyRules(args.POLICY_ID) + if err != nil { + return err + } + + printList(elblpr, len(elblpr), 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/monitor.go b/pkg/multicloud/hcs/shell/monitor.go new file mode 100644 index 0000000000..2ebfabd5fa --- /dev/null +++ b/pkg/multicloud/hcs/shell/monitor.go @@ -0,0 +1,38 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&cloudprovider.MetricListOptions{}, "metric-list", "List metrics", func(cli *hcs.SRegion, args *cloudprovider.MetricListOptions) error { + metrics, err := cli.GetClient().GetMetrics(args) + if err != nil { + return err + } + for i := range metrics { + log.Infof("metric: %s %s %s", metrics[i].Id, metrics[i].MetricType, metrics[i].Unit) + printList(metrics[i].Values, len(metrics[i].Values), 0, 0, []string{}) + } + return nil + }) + +} diff --git a/pkg/multicloud/hcs/shell/natgateway.go b/pkg/multicloud/hcs/shell/natgateway.go new file mode 100644 index 0000000000..55b0915d51 --- /dev/null +++ b/pkg/multicloud/hcs/shell/natgateway.go @@ -0,0 +1,141 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type NatGatewayOptions struct { + NatGatewayId string `help:"Nat Gateway Id"` + VpcId string `help:"Vpc Id"` + } + shellutils.R(&NatGatewayOptions{}, "nat-list", "List nat gateway", func(region *hcs.SRegion, args *NatGatewayOptions) error { + natGateways, err := region.GetNatGateways(args.VpcId) + if err != nil { + return err + } + printList(natGateways, 0, 0, 0, nil) + return nil + }) + + type NatGatewayIdOptions struct { + Id string `help:"Nat Id"` + } + + shellutils.R(&NatGatewayIdOptions{}, "nat-delete", "Delete nat gateways", func(cli *hcs.SRegion, args *NatGatewayIdOptions) error { + return cli.DeleteNatGateway(args.Id) + }) + + shellutils.R(&NatGatewayIdOptions{}, "nat-show", "Show nat gateways", func(cli *hcs.SRegion, args *NatGatewayIdOptions) error { + nat, err := cli.GetNatGateway(args.Id) + if err != nil { + return err + } + printObject(nat) + return nil + }) + + type SCreateDNatOptions struct { + GatewayId string `help:"Nat Gateway Id" positional:"true"` + Protocol string `help:"Protocol(tcp/udp)" positional:"true"` + ExternalIPId string `help:"External IP Id" positional:"true"` + ExternalPort int `help:"External Port" positional:"true"` + InternalIP string `help:"Internal IP" positional:"true"` + InternalPort int `help:"Nat Gateway Id" positional:"true"` + } + shellutils.R(&SCreateDNatOptions{}, "dnat-create", "Create dnat", func(region *hcs.SRegion, args *SCreateDNatOptions) error { + rule := cloudprovider.SNatDRule{ + Protocol: args.Protocol, + ExternalIPID: args.ExternalIPId, + ExternalPort: args.ExternalPort, + InternalIP: args.InternalIP, + InternalPort: args.InternalPort, + } + dnat, err := region.CreateNatDEntry(rule, args.GatewayId) + if err != nil { + return err + } + printObject(dnat) + return nil + }) + + type SCreateSNatOptions struct { + GatewayId string `help:"Nat Gateway Id" positional:"true"` + SourceCIDR string `help:"Source cidr" positional:"true"` + ExternalIPId string `help:"External IP Id" positional:"true"` + } + shellutils.R(&SCreateSNatOptions{}, "snat-create", "Create snat", func(region *hcs.SRegion, args *SCreateSNatOptions) error { + rule := cloudprovider.SNatSRule{ + SourceCIDR: args.SourceCIDR, + ExternalIPID: args.ExternalIPId, + } + snat, err := region.CreateNatSEntry(rule, args.GatewayId) + if err != nil { + return err + } + printObject(snat) + return nil + }) + + type SShowSNatOptions struct { + NatId string `help:"SNat Id" positional:"true"` + } + shellutils.R(&SShowSNatOptions{}, "snat-show", "Show snat", func(region *hcs.SRegion, args *SShowSNatOptions) error { + snat, err := region.GetNatSEntry(args.NatId) + if err != nil { + return err + } + printObject(snat) + return nil + }) + + type SShowDNatOptions struct { + NatId string `help:"DNat Id" positional:"true"` + } + shellutils.R(&SShowDNatOptions{}, "dnat-show", "Show dnat", func(region *hcs.SRegion, args *SShowDNatOptions) error { + dnat, err := region.GetNatDEntry(args.NatId) + if err != nil { + return err + } + printObject(dnat) + return nil + }) + + type SDeleteSNatOptions struct { + NatId string `help:"SNat Id" positional:"true"` + } + shellutils.R(&SDeleteSNatOptions{}, "snat-delete", "Delete snat", func(region *hcs.SRegion, args *SDeleteSNatOptions) error { + err := region.DeleteNatSEntry(args.NatId) + if err != nil { + return err + } + return nil + }) + + type SDeleteDNatOptions struct { + NatId string `help:"DNat Id" positional:"true"` + } + shellutils.R(&SDeleteDNatOptions{}, "dnat-delete", "Delete dnat", func(region *hcs.SRegion, args *SDeleteDNatOptions) error { + err := region.DeleteNatDEntry(args.NatId) + if err != nil { + return err + } + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/network.go b/pkg/multicloud/hcs/shell/network.go new file mode 100644 index 0000000000..e59e030ec9 --- /dev/null +++ b/pkg/multicloud/hcs/shell/network.go @@ -0,0 +1,48 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type NetworkListOptions struct { + Vpc string `help:"Vpc ID"` + } + shellutils.R(&NetworkListOptions{}, "network-list", "List subnets", func(cli *hcs.SRegion, args *NetworkListOptions) error { + networks, e := cli.GetNetwroks(args.Vpc) + if e != nil { + return e + } + printList(networks, 0, 0, 0, nil) + return nil + }) + + type NetworkIdOptions struct { + ID string + } + + shellutils.R(&NetworkIdOptions{}, "network-show", "Show subnets", func(cli *hcs.SRegion, args *NetworkIdOptions) error { + network, e := cli.GetNetwork(args.ID) + if e != nil { + return e + } + printObject(network) + return nil + }) + +} diff --git a/pkg/multicloud/hcs/shell/port.go b/pkg/multicloud/hcs/shell/port.go new file mode 100644 index 0000000000..5760c136d2 --- /dev/null +++ b/pkg/multicloud/hcs/shell/port.go @@ -0,0 +1,34 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type PortListOptions struct { + InstanceId string + } + shellutils.R(&PortListOptions{}, "port-list", "List ports", func(cli *hcs.SRegion, args *PortListOptions) error { + ports, err := cli.GetPorts(args.InstanceId) + if err != nil { + return err + } + printList(ports, 0, 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/printutils.go b/pkg/multicloud/hcs/shell/printutils.go new file mode 100644 index 0000000000..1b46ec704a --- /dev/null +++ b/pkg/multicloud/hcs/shell/printutils.go @@ -0,0 +1,25 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import "yunion.io/x/onecloud/pkg/util/printutils" + +func printList(data interface{}, total, offset, limit int, columns []string) { + printutils.PrintInterfaceList(data, total, offset, limit, columns) +} + +func printObject(obj interface{}) { + printutils.PrintInterfaceObject(obj) +} diff --git a/pkg/multicloud/hcs/shell/project.go b/pkg/multicloud/hcs/shell/project.go new file mode 100644 index 0000000000..86ceba6e7a --- /dev/null +++ b/pkg/multicloud/hcs/shell/project.go @@ -0,0 +1,33 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ProjectListOptions struct { + } + shellutils.R(&ProjectListOptions{}, "project-list", "List projects", func(cli *hcs.SRegion, args *ProjectListOptions) error { + regions, err := cli.GetClient().GetProjects() + if err != nil { + return err + } + printList(regions, 0, 0, 0, nil) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/redis.go b/pkg/multicloud/hcs/shell/redis.go new file mode 100644 index 0000000000..89bc5dbdaa --- /dev/null +++ b/pkg/multicloud/hcs/shell/redis.go @@ -0,0 +1,95 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ElasticcacheListOptions struct { + } + shellutils.R(&ElasticcacheListOptions{}, "redis-list", "List elasticcaches", func(cli *hcs.SRegion, args *ElasticcacheListOptions) error { + instances, e := cli.GetElasticCaches() + if e != nil { + return e + } + printList(instances, len(instances), 0, 0, []string{}) + return nil + }) + + type ElasitccacheInstanceTypeListOptions struct { + } + + shellutils.R(&ElasitccacheInstanceTypeListOptions{}, "redis-instance-type-list", "List elasticcache instancetypes", func(cli *hcs.SRegion, args *ElasitccacheInstanceTypeListOptions) error { + ret, e := cli.GetRedisInstnaceTypes() + if e != nil { + return e + } + printList(ret, 0, 0, 0, []string{}) + return nil + }) + + shellutils.R(&cloudprovider.SCloudElasticCacheInput{}, "redis-create", "Create elasticcache", func(cli *hcs.SRegion, args *cloudprovider.SCloudElasticCacheInput) error { + instance, e := cli.CreateElasticcache(args) + if e != nil { + return e + } + printObject(instance) + return nil + }) + + type ElasticcacheIdOptions struct { + ID string `help:"ID of instances to show"` + } + shellutils.R(&ElasticcacheIdOptions{}, "redis-show", "Show elasticcache", func(cli *hcs.SRegion, args *ElasticcacheIdOptions) error { + instance, err := cli.GetElasticCache(args.ID) + if err != nil { + return err + } + printObject(instance) + return nil + }) + + shellutils.R(&ElasticcacheIdOptions{}, "redis-delete", "Delete elasticcache", func(cli *hcs.SRegion, args *ElasticcacheIdOptions) error { + return cli.DeleteElasticcache(args.ID) + }) + + type ElasticcacheBackupsListOptions struct { + ID string `help:"ID of instances to show"` + StartTime string `help:"backup start time. format: 20060102150405"` + EndTime string `help:"backup end time. format: 20060102150405 "` + } + + shellutils.R(&ElasticcacheBackupsListOptions{}, "redis-backup-list", "List elasticcache backups", func(cli *hcs.SRegion, args *ElasticcacheBackupsListOptions) error { + backups, err := cli.GetElasticCacheBackups(args.ID, args.StartTime, args.EndTime) + if err != nil { + return err + } + printList(backups, 0, 0, 0, []string{}) + return nil + }) + + shellutils.R(&ElasticcacheIdOptions{}, "redis-parameter-list", "List elasticcache parameters", func(cli *hcs.SRegion, args *ElasticcacheIdOptions) error { + parameters, err := cli.GetElasticCacheParameters(args.ID) + if err != nil { + return err + } + printList(parameters, 0, 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/region.go b/pkg/multicloud/hcs/shell/region.go new file mode 100644 index 0000000000..2182317e83 --- /dev/null +++ b/pkg/multicloud/hcs/shell/region.go @@ -0,0 +1,30 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type RegionListOptions struct { + } + shellutils.R(&RegionListOptions{}, "region-list", "List regions", func(cli *hcs.SRegion, args *RegionListOptions) error { + regions := cli.GetClient().GetRegions() + printList(regions, 0, 0, 0, nil) + return nil + }) +} diff --git a/pkg/multicloud/hcs/shell/securitygroup.go b/pkg/multicloud/hcs/shell/securitygroup.go new file mode 100644 index 0000000000..c0e91e3b37 --- /dev/null +++ b/pkg/multicloud/hcs/shell/securitygroup.go @@ -0,0 +1,90 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/secrules" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type SecurityGroupListOptions struct { + VpcId string `help:"VPC ID"` + Name string `help:"Secgroup name"` + } + shellutils.R(&SecurityGroupListOptions{}, "security-group-list", "List security group", func(cli *hcs.SRegion, args *SecurityGroupListOptions) error { + secgrps, e := cli.GetSecurityGroups(args.VpcId) + if e != nil { + return e + } + printList(secgrps, 0, 0, 0, nil) + return nil + }) + + type SecurityGroupShowOptions struct { + ID string `help:"ID or name of security group"` + } + shellutils.R(&SecurityGroupShowOptions{}, "security-group-show", "Show details of a security group", func(cli *hcs.SRegion, args *SecurityGroupShowOptions) error { + secgrp, err := cli.GetSecurityGroupDetails(args.ID) + if err != nil { + return err + } + printObject(secgrp) + return nil + }) + + type SecurityGroupCreateOptions struct { + NAME string `help:"secgroup name"` + VPC string `help:"ID of VPC"` + Desc string `help:"description"` + } + shellutils.R(&SecurityGroupCreateOptions{}, "security-group-create", "Create security group", func(cli *hcs.SRegion, args *SecurityGroupCreateOptions) error { + result, err := cli.CreateSecurityGroup(args.VPC, args.NAME, args.Desc) + if err != nil { + return err + } + printObject(result) + return nil + }) + + type SecurityGroupRuleIdOptions struct { + ID string + } + + shellutils.R(&SecurityGroupRuleIdOptions{}, "security-group-rule-delete", "Delete security group rule", func(cli *hcs.SRegion, args *SecurityGroupRuleIdOptions) error { + return cli.DeleteSecurityGroupRule(args.ID) + }) + + type SecurityGroupRuleCreateOptions struct { + SECGROUP_ID string + RULE string + } + + shellutils.R(&SecurityGroupRuleCreateOptions{}, "security-group-rule-create", "Create security group rule", func(cli *hcs.SRegion, args *SecurityGroupRuleCreateOptions) error { + _rule, err := secrules.ParseSecurityRule(args.RULE) + if err != nil { + return errors.Wrapf(err, "invalid rule %s", args.RULE) + } + rule := cloudprovider.SecurityRule{ + SecurityRule: *_rule, + } + return cli.CreateSecurityGroupRule(args.SECGROUP_ID, rule) + }) + +} diff --git a/pkg/multicloud/hcs/shell/snapshot.go b/pkg/multicloud/hcs/shell/snapshot.go new file mode 100644 index 0000000000..d6570705b8 --- /dev/null +++ b/pkg/multicloud/hcs/shell/snapshot.go @@ -0,0 +1,63 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type SnapshotListOptions struct { + DiskId string `help:"Disk ID"` + Name string `help:"Snapshot Name"` + } + shellutils.R(&SnapshotListOptions{}, "snapshot-list", "List snapshot", func(cli *hcs.SRegion, args *SnapshotListOptions) error { + snapshots, err := cli.GetSnapshots(args.DiskId, args.Name) + if err != nil { + return err + } + printList(snapshots, 0, 0, 0, nil) + return nil + }) + + type SnapshotIdOptions struct { + ID string `help:"Snapshot ID"` + } + + shellutils.R(&SnapshotIdOptions{}, "snapshot-delete", "Delete snapshot", func(cli *hcs.SRegion, args *SnapshotIdOptions) error { + return cli.DeleteSnapshot(args.ID) + }) + + shellutils.R(&SnapshotIdOptions{}, "snapshot-show", "Show snapshot", func(cli *hcs.SRegion, args *SnapshotIdOptions) error { + ret, err := cli.GetSnapshot(args.ID) + if err != nil { + return err + } + printObject(ret) + return nil + }) + + type SnapshotCreateOptions struct { + DiskId string `help:"Disk ID"` + Name string `help:"Snapeshot Name"` + Desc string `help:"Snapshot Desc"` + } + + shellutils.R(&SnapshotCreateOptions{}, "snapshot-create", "Create snapshot", func(cli *hcs.SRegion, args *SnapshotCreateOptions) error { + _, err := cli.CreateSnapshot(args.DiskId, args.Name, args.Desc) + return err + }) +} diff --git a/pkg/multicloud/hcs/shell/vpc.go b/pkg/multicloud/hcs/shell/vpc.go new file mode 100644 index 0000000000..ba9257882a --- /dev/null +++ b/pkg/multicloud/hcs/shell/vpc.go @@ -0,0 +1,57 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type VpcListOptions struct { + } + shellutils.R(&VpcListOptions{}, "vpc-list", "List vpcs", func(cli *hcs.SRegion, args *VpcListOptions) error { + vpcs, err := cli.GetVpcs() + if err != nil { + return nil + } + printList(vpcs, 0, 0, 0, nil) + return nil + }) + + type VpcCreateOptions struct { + NAME string + CIDR string + Desc string + } + + shellutils.R(&VpcCreateOptions{}, "vpc-create", "Create vpc", func(cli *hcs.SRegion, args *VpcCreateOptions) error { + vpc, err := cli.CreateVpc(args.NAME, args.CIDR, args.Desc) + if err != nil { + return err + } + printObject(vpc) + return nil + }) + + type VpcIdOption struct { + ID string + } + + shellutils.R(&VpcIdOption{}, "vpc-delete", "Delete vpc", func(cli *hcs.SRegion, args *VpcIdOption) error { + return cli.DeleteVpc(args.ID) + }) + +} diff --git a/pkg/multicloud/hcs/shell/vpc_peer.go b/pkg/multicloud/hcs/shell/vpc_peer.go new file mode 100644 index 0000000000..b4010cd71c --- /dev/null +++ b/pkg/multicloud/hcs/shell/vpc_peer.go @@ -0,0 +1,52 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type VpcPeerListOptions struct { + VpcId string + } + shellutils.R(&VpcPeerListOptions{}, "vpc-peer-list", "List vpc peers", func(cli *hcs.SRegion, args *VpcPeerListOptions) error { + peers, err := cli.GetVpcPeerings(args.VpcId) + if err != nil { + return nil + } + printList(peers, 0, 0, 0, nil) + return nil + }) + + type VpcPeerIdOption struct { + ID string + } + + shellutils.R(&VpcPeerIdOption{}, "vpc-peer-show", "Show vpc peer", func(cli *hcs.SRegion, args *VpcPeerIdOption) error { + ret, err := cli.GetVpcPeering(args.ID) + if err != nil { + return err + } + printObject(ret) + return nil + }) + + shellutils.R(&VpcPeerIdOption{}, "vpc-peer-delete", "Delete vpc peer", func(cli *hcs.SRegion, args *VpcPeerIdOption) error { + return cli.DeleteVpcPeering(args.ID) + }) + +} diff --git a/pkg/multicloud/hcs/shell/zone.go b/pkg/multicloud/hcs/shell/zone.go new file mode 100644 index 0000000000..e4ddf4a1af --- /dev/null +++ b/pkg/multicloud/hcs/shell/zone.go @@ -0,0 +1,33 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/hcs" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ZoneListOptions struct { + } + shellutils.R(&ZoneListOptions{}, "zone-list", "List zones", func(cli *hcs.SRegion, args *ZoneListOptions) error { + zones, err := cli.GetZones() + if err != nil { + return nil + } + printList(zones, 0, 0, 0, nil) + return nil + }) +} diff --git a/pkg/multicloud/hcs/signer.go b/pkg/multicloud/hcs/signer.go new file mode 100644 index 0000000000..04aade025e --- /dev/null +++ b/pkg/multicloud/hcs/signer.go @@ -0,0 +1,244 @@ +// HWS API Gateway Signature +// based on https://github.com/datastream/aws/blob/master/signv4.go +// Copyright (c) 2014, Xianjie + +package hcs + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "fmt" + "io/ioutil" + "net/http" + "sort" + "strings" + "time" +) + +const ( + BasicDateFormat = "20060102T150405Z" + Algorithm = "SDK-HMAC-SHA256" + HeaderXDate = "X-Sdk-Date" + HeaderHost = "host" + HeaderAuthorization = "Authorization" + HeaderContentSha256 = "X-Sdk-Content-Sha256" +) + +func shouldEscape(c byte) bool { + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c == '-' || c == '~' || c == '.' { + return false + } + return true +} +func escape(s string) string { + hexCount := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c) { + hexCount++ + } + } + + if hexCount == 0 { + return s + } + + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case shouldEscape(c): + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + default: + t[j] = s[i] + j++ + } + } + return string(t) +} + +func hmacsha256(key []byte, data string) ([]byte, error) { + h := hmac.New(sha256.New, []byte(key)) + if _, err := h.Write([]byte(data)); err != nil { + return nil, err + } + return h.Sum(nil), nil +} + +// Build a CanonicalRequest from a regular request string +// +// CanonicalRequest = +// HTTPRequestMethod + '\n' + +// CanonicalURI + '\n' + +// CanonicalQueryString + '\n' + +// CanonicalHeaders + '\n' + +// SignedHeaders + '\n' + +// HexEncode(Hash(RequestPayload)) +func CanonicalRequest(r *http.Request, signedHeaders []string) (string, error) { + var hexencode string + var err error + if hex := r.Header.Get(HeaderContentSha256); hex != "" { + hexencode = hex + } else { + data, err := RequestPayload(r) + if err != nil { + return "", err + } + hexencode, err = HexEncodeSHA256Hash(data) + if err != nil { + return "", err + } + } + return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, CanonicalURI(r), CanonicalQueryString(r), CanonicalHeaders(r, signedHeaders), strings.Join(signedHeaders, ";"), hexencode), err +} + +// CanonicalURI returns request uri +func CanonicalURI(r *http.Request) string { + pattens := strings.Split(r.URL.Path, "/") + var uri []string + for _, v := range pattens { + uri = append(uri, escape(v)) + } + urlpath := strings.Join(uri, "/") + if len(urlpath) == 0 || urlpath[len(urlpath)-1] != '/' { + urlpath = urlpath + "/" + } + return urlpath +} + +// CanonicalQueryString +func CanonicalQueryString(r *http.Request) string { + var keys []string + query := r.URL.Query() + for key := range query { + keys = append(keys, key) + } + sort.Strings(keys) + var a []string + for _, key := range keys { + k := escape(key) + sort.Strings(query[key]) + for _, v := range query[key] { + kv := fmt.Sprintf("%s=%s", k, escape(v)) + a = append(a, kv) + } + } + queryStr := strings.Join(a, "&") + r.URL.RawQuery = queryStr + return queryStr +} + +// CanonicalHeaders +func CanonicalHeaders(r *http.Request, signerHeaders []string) string { + var a []string + header := make(map[string][]string) + for k, v := range r.Header { + header[strings.ToLower(k)] = v + } + for _, key := range signerHeaders { + value := header[key] + if strings.EqualFold(key, HeaderHost) { + value = []string{r.Host} + } + sort.Strings(value) + for _, v := range value { + a = append(a, key+":"+strings.TrimSpace(v)) + } + } + return fmt.Sprintf("%s\n", strings.Join(a, "\n")) +} + +// SignedHeaders +func SignedHeaders(r *http.Request) []string { + var a []string + for key := range r.Header { + a = append(a, strings.ToLower(key)) + } + sort.Strings(a) + return a +} + +// RequestPayload +func RequestPayload(r *http.Request) ([]byte, error) { + if r.Body == nil { + return []byte(""), nil + } + b, err := ioutil.ReadAll(r.Body) + if err != nil { + return []byte(""), err + } + r.Body = ioutil.NopCloser(bytes.NewBuffer(b)) + return b, err +} + +// Create a "String to Sign". +func StringToSign(canonicalRequest string, t time.Time) (string, error) { + hash := sha256.New() + _, err := hash.Write([]byte(canonicalRequest)) + if err != nil { + return "", err + } + return fmt.Sprintf("%s\n%s\n%x", + Algorithm, t.UTC().Format(BasicDateFormat), hash.Sum(nil)), nil +} + +// Create the HWS Signature. +func SignStringToSign(stringToSign string, signingKey []byte) (string, error) { + hm, err := hmacsha256(signingKey, stringToSign) + return fmt.Sprintf("%x", hm), err +} + +// HexEncodeSHA256Hash returns hexcode of sha256 +func HexEncodeSHA256Hash(body []byte) (string, error) { + hash := sha256.New() + if body == nil { + body = []byte("") + } + _, err := hash.Write(body) + return fmt.Sprintf("%x", hash.Sum(nil)), err +} + +// Get the finalized value for the "Authorization" header. The signature parameter is the output from SignStringToSign +func AuthHeaderValue(signature, accessKey string, signedHeaders []string) string { + return fmt.Sprintf("%s Access=%s, SignedHeaders=%s, Signature=%s", Algorithm, accessKey, strings.Join(signedHeaders, ";"), signature) +} + +// Signature HWS meta +type Signer struct { + Key string + Secret string +} + +// SignRequest set Authorization header +func (s *Signer) Sign(r *http.Request) error { + var t time.Time + var err error + var dt string + if dt = r.Header.Get(HeaderXDate); dt != "" { + t, err = time.Parse(BasicDateFormat, dt) + } + if err != nil || dt == "" { + t = time.Now() + r.Header.Set(HeaderXDate, t.UTC().Format(BasicDateFormat)) + } + signedHeaders := SignedHeaders(r) + canonicalRequest, err := CanonicalRequest(r, signedHeaders) + if err != nil { + return err + } + stringToSign, err := StringToSign(canonicalRequest, t) + if err != nil { + return err + } + signature, err := SignStringToSign(stringToSign, []byte(s.Secret)) + if err != nil { + return err + } + authValue := AuthHeaderValue(signature, s.Key, signedHeaders) + r.Header.Set(HeaderAuthorization, authValue) + return nil +} diff --git a/pkg/multicloud/hcs/snapshot.go b/pkg/multicloud/hcs/snapshot.go new file mode 100644 index 0000000000..f765f34dc2 --- /dev/null +++ b/pkg/multicloud/hcs/snapshot.go @@ -0,0 +1,192 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +/* +限制： +https://support.huaweicloud.com/api-evs/zh-cn_topic_0058762427.html +1. 从快照创建云硬盘时，volume_type字段必须和快照源云硬盘保持一致。 +2. 当指定的云硬盘类型在avaliability_zone内不存在时，则创建云硬盘失败。 +*/ + +type SnapshotStatusType string + +const ( + SnapshotStatusCreating SnapshotStatusType = "creating" + SnapshotStatusAvailable SnapshotStatusType = "available" // 云硬盘快照创建成功，可以使用。 + SnapshotStatusError SnapshotStatusType = "error" // 云硬盘快照在创建过程中出现错误。 + SnapshotStatusDeleting SnapshotStatusType = "deleting" // 云硬盘快照处于正在删除的过程中。 + SnapshotStatusErrorDeleting SnapshotStatusType = "error_deleting" // 云硬盘快照在删除过程中出现错误 + SnapshotStatusRollbacking SnapshotStatusType = "rollbacking" // 云硬盘快照处于正在回滚数据的过程中。 + SnapshotStatusBackingUp SnapshotStatusType = "backing-up" // 通过快照创建备份，快照状态就会变为backing-up +) + +type Metadata struct { + SystemEnableActive string `json:"__system__enableActive"` // 如果为true。则表明是系统盘快照 +} + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0051408624.html +type SSnapshot struct { + multicloud.SResourceBase + multicloud.HcsTags + region *SRegion + + Metadata Metadata `json:"metadata"` + CreatedAt string `json:"created_at"` + Description string `json:"description"` + Id string `json:"id"` + Name string `json:"name"` + OSExtendedSnapshotAttributesProgress string `json:"os-extended-snapshot-attributes:progress"` + OSExtendedSnapshotAttributesProjectId string `json:"os-extended-snapshot-attributes:project_id"` + Size int32 `json:"size"` // GB + Status string `json:"status"` + UpdatedAt string `json:"updated_at"` + VolumeId string `json:"volume_id"` +} + +func (self *SSnapshot) GetId() string { + return self.Id +} + +func (self *SSnapshot) GetName() string { + return self.Name +} + +func (self *SSnapshot) GetGlobalId() string { + return self.Id +} + +func (self *SSnapshot) GetStatus() string { + switch SnapshotStatusType(self.Status) { + case SnapshotStatusAvailable: + return api.SNAPSHOT_READY + case SnapshotStatusCreating: + return api.SNAPSHOT_CREATING + case SnapshotStatusDeleting: + return api.SNAPSHOT_DELETING + case SnapshotStatusErrorDeleting, SnapshotStatusError: + return api.SNAPSHOT_FAILED + case SnapshotStatusRollbacking: + return api.SNAPSHOT_ROLLBACKING + default: + return api.SNAPSHOT_UNKNOWN + } +} + +func (self *SSnapshot) Refresh() error { + ret, err := self.region.GetSnapshot(self.Id) + if err != nil { + return err + } + return jsonutils.Update(self, ret) +} + +func (self *SSnapshot) GetSizeMb() int32 { + return self.Size * 1024 +} + +func (self *SSnapshot) GetDiskId() string { + return self.VolumeId +} + +func (self *SSnapshot) GetDiskType() string { + if self.Metadata.SystemEnableActive == "true" { + return api.DISK_TYPE_SYS + } + return api.DISK_TYPE_DATA +} + +func (self *SSnapshot) Delete() error { + return self.region.DeleteSnapshot(self.Id) +} + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0051408627.html +func (self *SRegion) GetSnapshots(diskId string, name string) ([]SSnapshot, error) { + params := url.Values{} + if len(diskId) > 0 { + params.Set("volume_id", diskId) + } + + if len(name) > 0 { + params.Set("name", name) + } + ret := []SSnapshot{} + return ret, self.evsList("snapshots", params, &ret) +} + +func (self *SRegion) GetSnapshot(id string) (*SSnapshot, error) { + ret := &SSnapshot{region: self} + res := fmt.Sprintf("snapshots/%s", id) + return ret, self.evsGet(res, ret) +} + +// 不能删除以autobk_snapshot_为前缀的快照。 +// 当快照状态为available、error状态时，才可以删除。 +func (self *SRegion) DeleteSnapshot(id string) error { + res := fmt.Sprintf("snapshots/%s", id) + return self.evsDelete(res) +} + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0051408624.html +// 目前已设置force字段。云硬盘处于挂载状态时，能强制创建快照。 +func (self *SRegion) CreateSnapshot(diskId, name, desc string) (*SSnapshot, error) { + params := map[string]interface{}{ + "snapshot": map[string]interface{}{ + "name": name, + "description": desc, + "volume_id": diskId, + "force": true, + }, + } + ret := &SSnapshot{region: self} + return ret, self.evsCreate("snapshots", params, ret) +} + +func (self *SSnapshot) GetProjectId() string { + return "" +} + +func (self *SRegion) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { + snapshots, err := self.GetSnapshots("", "") + if err != nil { + return nil, errors.Wrapf(err, "GetSnapshots") + } + ret := []cloudprovider.ICloudSnapshot{} + for i := 0; i < len(snapshots); i += 1 { + snapshots[i].region = self + ret = append(ret, &snapshots[i]) + } + return ret, nil +} + +func (self *SRegion) GetISnapshotById(id string) (cloudprovider.ICloudSnapshot, error) { + ret, err := self.GetSnapshot(id) + if err != nil { + return nil, err + } + return ret, nil +} diff --git a/pkg/multicloud/hcs/storage.go b/pkg/multicloud/hcs/storage.go new file mode 100644 index 0000000000..26400473e8 --- /dev/null +++ b/pkg/multicloud/hcs/storage.go @@ -0,0 +1,134 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SStorage struct { + multicloud.SStorageBase + multicloud.HuaweiTags + + zone *SZone + storageType string // volume_type 目前支持“SSD”，“SAS”和“SATA”三种 +} + +func (self *SStorage) GetId() string { + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId(), self.storageType) +} + +func (self *SStorage) GetName() string { + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), self.storageType) +} + +func (self *SStorage) GetGlobalId() string { + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), self.storageType) +} + +func (self *SStorage) GetStatus() string { + return api.STORAGE_ONLINE +} + +func (self *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache { + cache := &SStoragecache{region: self.zone.region} + return cache +} + +func (self *SStorage) GetIZone() cloudprovider.ICloudZone { + return self.zone +} + +func (self *SStorage) GetIDisks() ([]cloudprovider.ICloudDisk, error) { + disks, err := self.zone.region.GetDisks(self.zone.GetId()) + if err != nil { + return nil, err + } + + // 按storage type 过滤出disk + filtedDisks := make([]SDisk, 0) + for i := range disks { + disk := disks[i] + if disk.VolumeType == self.storageType { + filtedDisks = append(filtedDisks, disk) + } + } + + idisks := make([]cloudprovider.ICloudDisk, len(filtedDisks)) + for i := 0; i < len(filtedDisks); i += 1 { + filtedDisks[i].region = self.zone.region + idisks[i] = &filtedDisks[i] + } + return idisks, nil +} + +func (self *SStorage) GetStorageType() string { + return self.storageType +} + +func (self *SStorage) GetMediumType() string { + if self.storageType == api.STORAGE_HUAWEI_SSD { + return api.DISK_TYPE_SSD + } + return api.DISK_TYPE_ROTATE +} + +func (self *SStorage) GetCapacityMB() int64 { + return 0 // unlimited +} + +func (self *SStorage) GetCapacityUsedMB() int64 { + return 0 +} + +func (self *SStorage) GetStorageConf() jsonutils.JSONObject { + conf := jsonutils.NewDict() + return conf +} + +func (self *SStorage) GetEnabled() bool { + return true +} + +func (self *SStorage) CreateIDisk(conf *cloudprovider.DiskCreateConfig) (cloudprovider.ICloudDisk, error) { + disk, err := self.zone.region.CreateDisk(self.zone.GetId(), self.storageType, conf.Name, conf.SizeGb, "", conf.Desc, conf.ProjectId) + if err != nil { + return nil, errors.Wrapf(err, "CreateDisk") + } + return disk, nil +} + +func (self *SStorage) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) { + disk, err := self.zone.region.GetDisk(id) + if err != nil { + return nil, err + } + return disk, nil +} + +func (self *SStorage) GetMountPoint() string { + return "" +} + +func (self *SStorage) IsSysDiskStore() bool { + return true +} diff --git a/pkg/multicloud/hcs/storagecache.go b/pkg/multicloud/hcs/storagecache.go new file mode 100644 index 0000000000..e3fb0e6dc9 --- /dev/null +++ b/pkg/multicloud/hcs/storagecache.go @@ -0,0 +1,253 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "context" + "fmt" + "math" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/image" + "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/qemuimg" +) + +type SStoragecache struct { + multicloud.SResourceBase + multicloud.HuaweiTags + region *SRegion +} + +func GetBucketName(regionId string, imageId string) string { + return fmt.Sprintf("imgcache-%s-%s", strings.ToLower(regionId), imageId) +} + +func (self *SStoragecache) GetId() string { + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) +} + +func (self *SStoragecache) GetName() string { + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) +} + +func (self *SStoragecache) GetGlobalId() string { + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) +} + +func (self *SStoragecache) GetStatus() string { + return "available" +} + +func (self *SStoragecache) GetICloudImages() ([]cloudprovider.ICloudImage, error) { + imagesSelf, err := self.region.GetImages("", ImageOwnerSelf, "", EnvFusionCompute) + if err != nil { + return nil, errors.Wrapf(err, "GetImages") + } + + ret := []cloudprovider.ICloudImage{} + for i := range imagesSelf { + imagesSelf[i].cache = self + ret = append(ret, &imagesSelf[i]) + } + return ret, nil +} + +func (self *SStoragecache) GetICustomizedCloudImages() ([]cloudprovider.ICloudImage, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (self *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { + image, err := self.region.GetImage(extId) + if err != nil { + return nil, errors.Wrap(err, "self.region.GetImage") + } + image.cache = self + return image, nil +} + +func (self *SStoragecache) GetPath() string { + return "" +} + +// 目前支持使用vhd、zvhd、vmdk、qcow2、raw、zvhd2、vhdx、qcow、vdi或qed格式镜像文件创建私有镜像。 +// 快速通道功能可快速完成镜像制作，但镜像文件需转换为raw或zvhd2格式并完成镜像优化。 +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0083905788.html +func (self *SStoragecache) CreateIImage(snapshotId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { + if imageId, err := self.region.createIImage(snapshotId, imageName, imageDesc); err != nil { + return nil, err + } else if image, err := self.region.GetImage(imageId); err != nil { + return nil, err + } else { + image.cache = self + iimage := make([]cloudprovider.ICloudImage, 1) + iimage[0] = image + if err := cloudprovider.WaitStatus(iimage[0], "avaliable", 15*time.Second, 3600*time.Second); err != nil { + return nil, err + } + return iimage[0], nil + } +} + +func (self *SStoragecache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error) { + return self.downloadImage(userCred, imageId, extId) +} + +func (self *SStoragecache) downloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (self *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, callback func(progress float32)) (string, error) { + return self.uploadImage(ctx, userCred, image, callback) +} + +func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, callback func(progress float32)) (string, error) { + bucketName := GetBucketName(self.region.GetId(), image.ImageId) + + exist, _ := self.region.IBucketExist(bucketName) + if !exist { + err := self.region.CreateIBucket(bucketName, "", "") + if err != nil { + return "", errors.Wrap(err, "CreateIBucket") + } + } + defer self.region.DeleteIBucket(bucketName) + + // upload to huawei cloud + s := auth.GetAdminSession(ctx, options.Options.Region) + meta, reader, sizeByte, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) + if err != nil { + return "", errors.Wrap(err, "Images.Download") + } + log.Debugf("Images meta data %s", meta) + + minDiskMB, _ := meta.Int("min_disk") + minDiskGB := int64(math.Ceil(float64(minDiskMB) / 1024)) + // 在使用OBS桶的外部镜像文件制作镜像时生效且为必选字段。取值为40～1024GB。 + if minDiskGB < 40 { + minDiskGB = 40 + } else if minDiskGB > 1024 { + minDiskGB = 1024 + } + + bucket, err := self.region.GetIBucketByName(bucketName) + if err != nil { + return "", errors.Wrapf(err, "GetIBucketByName %s", bucketName) + } + + body := multicloud.NewProgress(sizeByte, 95, reader, callback) + err = cloudprovider.UploadObject(context.Background(), bucket, image.ImageId, 0, body, sizeByte, "", "", nil, false) + if err != nil { + return "", errors.Wrap(err, "cloudprovider.UploadObject") + } + + defer bucket.DeleteObject(context.Background(), image.ImageId) + + // check image name, avoid name conflict + imageBaseName := image.ImageId + if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { + imageBaseName = fmt.Sprintf("img%s", image.ImageId) + } + imageName := imageBaseName + nameIdx := 1 + + for { + _, err = self.region.GetImageByName(imageName) + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + break + } else { + return "", err + } + } + + imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx) + nameIdx += 1 + log.Debugf("uploadImage Match remote name %s", imageName) + } + + err = self.region.ImportImageJob(imageName, image.OsDistribution, image.OsVersion, image.OsArch, bucketName, image.ImageId, int64(minDiskGB)) + if err != nil { + return "", errors.Wrapf(err, "ImportImageJob") + } + + if callback != nil { + callback(100) + } + return "", cloudprovider.ErrNotImplemented +} + +func (self *SRegion) getStoragecache() *SStoragecache { + return &SStoragecache{region: self} +} + +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0020092109.html +func (self *SRegion) createIImage(snapshotId, imageName, imageDesc string) (string, error) { + snapshot, err := self.GetSnapshot(snapshotId) + if err != nil { + return "", err + } + + disk, err := self.GetDisk(snapshot.VolumeId) + if err != nil { + return "", err + } + + if disk.GetDiskType() != api.DISK_TYPE_SYS { + return "", fmt.Errorf("disk type err, expected disk type %s", api.DISK_TYPE_SYS) + } + + if len(disk.Attachments) == 0 { + return "", fmt.Errorf("disk is not attached.") + } + + params := map[string]interface{}{ + "name": imageName, + "description": imageDesc, + "instance_id": disk.Attachments[0].ServerId, + } + + ret := struct { + ImageId string + }{} + err = self.imsPerform("cloudimages", "action", params, &ret) + if err != nil { + return "", err + } + return ret.ImageId, nil +} + +func (self *SRegion) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, error) { + storageCache := self.getStoragecache() + return []cloudprovider.ICloudStoragecache{storageCache}, nil +} + +func (self *SRegion) GetIStoragecacheById(idstr string) (cloudprovider.ICloudStoragecache, error) { + storageCache := self.getStoragecache() + if storageCache.GetGlobalId() == idstr { + return storageCache, nil + } + return nil, cloudprovider.ErrNotFound +} diff --git a/pkg/multicloud/hcs/vpc.go b/pkg/multicloud/hcs/vpc.go new file mode 100644 index 0000000000..407c67c275 --- /dev/null +++ b/pkg/multicloud/hcs/vpc.go @@ -0,0 +1,312 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SVpc struct { + multicloud.SVpc + multicloud.HcsTags + + region *SRegion + + Id string + Name string + CIDR string + Status string + EnterpriseProjectId string +} + +func (self *SRegion) GetVpcs() ([]SVpc, error) { + ret := []SVpc{} + return ret, self.vpcList("vpcs", nil, &ret) +} + +func (self *SVpc) GetId() string { + return self.Id +} + +func (self *SVpc) GetName() string { + if len(self.Name) > 0 { + return self.Name + } + return self.Id +} + +func (self *SVpc) GetGlobalId() string { + return self.Id +} + +func (self *SVpc) GetStatus() string { + return api.VPC_STATUS_AVAILABLE +} + +func (self *SVpc) Refresh() error { + ret, err := self.region.GetVpc(self.GetId()) + if err != nil { + return err + } + return jsonutils.Update(self, ret) +} + +func (self *SVpc) GetRegion() cloudprovider.ICloudRegion { + return self.region +} + +func (self *SVpc) GetIsDefault() bool { + return false +} + +func (self *SVpc) GetCidrBlock() string { + return self.CIDR +} + +func (self *SVpc) GetIWires() ([]cloudprovider.ICloudWire, error) { + wire := &SWire{region: self.region, vpc: self} + return []cloudprovider.ICloudWire{wire}, nil +} + +func (self *SVpc) GetIWireById(id string) (cloudprovider.ICloudWire, error) { + wires, err := self.GetIWires() + if err != nil { + return nil, err + } + for i := range wires { + if wires[i].GetGlobalId() == id { + return wires[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, error) { + secgroups, err := self.region.GetSecurityGroups(self.Id) + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudSecurityGroup{} + for i := range secgroups { + secgroups[i].region = self.region + ret = append(ret, &secgroups[i]) + } + return ret, nil +} + +func (self *SVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) { + rtbs, err := self.region.GetRouteTables(self.Id) + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudRouteTable{} + for i := range rtbs { + rtbs[i].vpc = self + ret = append(ret, &rtbs[i]) + } + return ret, nil +} + +func (self *SVpc) GetIRouteTableById(routeTableId string) (cloudprovider.ICloudRouteTable, error) { + rtb, err := self.region.GetRouteTable(routeTableId) + if err != nil { + return nil, err + } + rtb.vpc = self + return rtb, nil +} + +func (self *SVpc) Delete() error { + return self.region.DeleteVpc(self.GetId()) +} + +func (self *SVpc) GetINatGateways() ([]cloudprovider.ICloudNatGateway, error) { + nats, err := self.region.GetNatGateways(self.GetId()) + if err != nil { + return nil, err + } + ret := make([]cloudprovider.ICloudNatGateway, len(nats)) + for i := 0; i < len(nats); i++ { + nats[i].region = self.region + ret[i] = &nats[i] + } + return ret, nil +} + +func (self *SVpc) GetICloudVpcPeeringConnections() ([]cloudprovider.ICloudVpcPeeringConnection, error) { + svpcPCs, err := self.getVpcPeeringConnections() + if err != nil { + return nil, errors.Wrap(err, "self.getVpcPeeringConnections()") + } + ivpcPCs := []cloudprovider.ICloudVpcPeeringConnection{} + for i := range svpcPCs { + ivpcPCs = append(ivpcPCs, &svpcPCs[i]) + } + return ivpcPCs, nil +} + +func (self *SVpc) GetICloudAccepterVpcPeeringConnections() ([]cloudprovider.ICloudVpcPeeringConnection, error) { + svpcPCs, err := self.getAccepterVpcPeeringConnections() + if err != nil { + return nil, errors.Wrap(err, "self.getAccepterVpcPeeringConnections()") + } + ivpcPCs := []cloudprovider.ICloudVpcPeeringConnection{} + for i := range svpcPCs { + ivpcPCs = append(ivpcPCs, &svpcPCs[i]) + } + return ivpcPCs, nil +} + +func (self *SVpc) GetICloudVpcPeeringConnectionById(id string) (cloudprovider.ICloudVpcPeeringConnection, error) { + svpcPC, err := self.getVpcPeeringConnectionById(id) + if err != nil { + return nil, errors.Wrapf(err, "self.getVpcPeeringConnectionById(%s)", id) + } + return svpcPC, nil +} + +func (self *SVpc) CreateICloudVpcPeeringConnection(opts *cloudprovider.VpcPeeringConnectionCreateOptions) (cloudprovider.ICloudVpcPeeringConnection, error) { + svpcPC, err := self.region.CreateVpcPeering(self.GetId(), opts) + if err != nil { + return nil, errors.Wrapf(err, "self.region.CreateVpcPeering(%s,%s)", self.GetId(), jsonutils.Marshal(opts).String()) + } + svpcPC.vpc = self + return svpcPC, nil +} + +func (self *SVpc) AcceptICloudVpcPeeringConnection(id string) error { + vpcPC, err := self.getVpcPeeringConnectionById(id) + if err != nil { + return errors.Wrapf(err, "self.getVpcPeeringConnectionById(%s)", id) + } + if vpcPC.GetStatus() == api.VPC_PEERING_CONNECTION_STATUS_ACTIVE { + return nil + } + if vpcPC.GetStatus() == api.VPC_PEERING_CONNECTION_STATUS_UNKNOWN { + return errors.Wrapf(cloudprovider.ErrInvalidStatus, "vpcPC: %s", jsonutils.Marshal(vpcPC).String()) + } + err = self.region.AcceptVpcPeering(id) + if err != nil { + return errors.Wrapf(err, "self.region.AcceptVpcPeering(%s)", id) + } + return nil +} + +func (self *SVpc) GetAuthorityOwnerId() string { + return self.region.client.projectId +} + +func (self *SRegion) GetVpc(id string) (*SVpc, error) { + vpc := &SVpc{region: self} + res := fmt.Sprintf("vpcs/%s", id) + return vpc, self.vpcGet(res, vpc) +} + +func (self *SRegion) DeleteVpc(id string) error { + if id != "default" { + secgroups, err := self.GetSecurityGroups(id) + if err != nil { + return errors.Wrap(err, "GetSecurityGroups") + } + for _, secgroup := range secgroups { + err = self.DeleteSecurityGroup(secgroup.Id) + if err != nil { + return errors.Wrapf(err, "DeleteSecurityGroup(%s)", secgroup.Id) + } + } + } + res := fmt.Sprintf("vpcs/%s", id) + return self.vpcDelete(res) +} + +func (self *SRegion) CreateVpc(name, cidr, desc string) (*SVpc, error) { + params := map[string]interface{}{ + "vpc": map[string]string{ + "name": name, + "cidr": cidr, + "description": desc, + }, + } + vpc := &SVpc{region: self} + return vpc, self.vpcCreate("vpcs", params, vpc) +} + +func (self *SRegion) GetIVpcs() ([]cloudprovider.ICloudVpc, error) { + vpcs, err := self.GetVpcs() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudVpc{} + for i := range vpcs { + vpcs[i].region = self + ret = append(ret, &vpcs[i]) + } + return ret, nil +} + +func (self *SRegion) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { + vpc, err := self.GetVpc(id) + if err != nil { + return nil, err + } + vpc.region = self + return vpc, nil +} + +func (self *SVpc) getVpcPeeringConnections() ([]SVpcPeering, error) { + svpcPeerings, err := self.region.GetVpcPeerings(self.GetId()) + if err != nil { + return nil, errors.Wrapf(err, "self.region.GetVpcPeerings(%s)", self.GetId()) + } + vpcPCs := []SVpcPeering{} + for i := range svpcPeerings { + if svpcPeerings[i].GetVpcId() == self.GetId() { + svpcPeerings[i].vpc = self + vpcPCs = append(vpcPCs, svpcPeerings[i]) + } + } + return vpcPCs, nil +} + +func (self *SVpc) getAccepterVpcPeeringConnections() ([]SVpcPeering, error) { + svpcPeerings, err := self.region.GetVpcPeerings(self.GetId()) + if err != nil { + return nil, errors.Wrapf(err, "self.region.GetVpcPeerings(%s)", self.GetId()) + } + vpcPCs := []SVpcPeering{} + for i := range svpcPeerings { + if svpcPeerings[i].GetPeerVpcId() == self.GetId() { + svpcPeerings[i].vpc = self + vpcPCs = append(vpcPCs, svpcPeerings[i]) + } + } + return vpcPCs, nil +} + +func (self *SVpc) getVpcPeeringConnectionById(id string) (*SVpcPeering, error) { + svpcPC, err := self.region.GetVpcPeering(id) + if err != nil { + return nil, errors.Wrapf(err, "self.region.GetVpcPeering(%s)", id) + } + svpcPC.vpc = self + return svpcPC, nil +} diff --git a/pkg/multicloud/hcs/vpc_peering.go b/pkg/multicloud/hcs/vpc_peering.go new file mode 100644 index 0000000000..5025cf5da3 --- /dev/null +++ b/pkg/multicloud/hcs/vpc_peering.go @@ -0,0 +1,143 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + "net/url" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type RequestVpcInfo struct { + VpcId string `json:"vpc_id"` + TenantId string `json:"tenant_id"` +} + +type AcceptVpcInfo struct { + VpcId string `json:"vpc_id"` + TenantId string `json:"tenant_id"` +} + +type SVpcPeering struct { + multicloud.SResourceBase + multicloud.HuaweiTags + vpc *SVpc + + RequestVpcInfo RequestVpcInfo `json:"request_vpc_info"` + AcceptVpcInfo AcceptVpcInfo `json:"accept_vpc_info"` + Name string `json:"name"` + Id string `json:"id"` + Status string `json:"status"` +} + +func (self *SRegion) GetVpcPeerings(vpcId string) ([]SVpcPeering, error) { + params := url.Values{} + if len(vpcId) > 0 { + params.Set("vpc_id", vpcId) + } + ret := []SVpcPeering{} + return ret, self.list("vpc", "v2.0", "vpc/peerings", params, &ret) +} + +func (self *SRegion) GetVpcPeering(id string) (*SVpcPeering, error) { + ret := SVpcPeering{} + resource := fmt.Sprintf("vpc/peerings/%s", id) + return &ret, self.get("vpc", "v2.0", resource, &ret) +} + +func (self *SRegion) CreateVpcPeering(vpcId string, opts *cloudprovider.VpcPeeringConnectionCreateOptions) (*SVpcPeering, error) { + params := map[string]interface{}{ + "peering": map[string]interface{}{ + "name": opts.Name, + "request_vpc_info": map[string]interface{}{ + "vpc_id": vpcId, + "tenant_id": self.client.projectId, + }, + "accept_vpc_info": map[string]interface{}{ + "vpc_id": opts.PeerVpcId, + "tenant_id": opts.PeerAccountId, + }, + }, + } + ret := &SVpcPeering{} + return ret, self.create("vpc", "v2.0", "vpc/peerings", params, ret) +} + +func (self *SRegion) AcceptVpcPeering(id string) error { + res := fmt.Sprintf("vpc/peerings/%s/accept", id) + return self.update("vpc", "v2.0", res, nil) +} + +func (self *SRegion) DeleteVpcPeering(id string) error { + res := fmt.Sprintf("vpc/peerings/%s", id) + return self.delete("vpc", "v2.0", res) +} + +func (self *SVpcPeering) GetId() string { + return self.Id +} + +func (self *SVpcPeering) GetName() string { + return self.Name +} + +func (self *SVpcPeering) GetGlobalId() string { + return self.GetId() +} + +func (self *SVpcPeering) GetStatus() string { + switch self.Status { + case "PENDING_ACCEPTANCE": + return api.VPC_PEERING_CONNECTION_STATUS_PENDING_ACCEPT + case "ACTIVE": + return api.VPC_PEERING_CONNECTION_STATUS_ACTIVE + default: + return api.VPC_PEERING_CONNECTION_STATUS_UNKNOWN + } +} + +func (self *SVpcPeering) Refresh() error { + peer, err := self.vpc.region.GetVpcPeering(self.Id) + if err != nil { + return errors.Wrapf(err, "self.region.GetVpcPeering(%s)", self.Id) + } + return jsonutils.Update(self, peer) +} + +func (self *SVpcPeering) GetVpcId() string { + return self.RequestVpcInfo.VpcId +} + +func (self *SVpcPeering) GetPeerVpcId() string { + return self.AcceptVpcInfo.VpcId +} + +func (self *SVpcPeering) GetPeerAccountId() string { + return self.AcceptVpcInfo.TenantId +} + +func (self *SVpcPeering) GetEnabled() bool { + return true +} + +func (self *SVpcPeering) Delete() error { + return self.vpc.region.DeleteVpcPeering(self.Id) +} diff --git a/pkg/multicloud/hcs/wire.go b/pkg/multicloud/hcs/wire.go new file mode 100644 index 0000000000..d9742e9407 --- /dev/null +++ b/pkg/multicloud/hcs/wire.go @@ -0,0 +1,127 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/netutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +// 华为云的子网有点特殊。子网在整个region可用。 +type SWire struct { + multicloud.SResourceBase + multicloud.HcsTags + region *SRegion + vpc *SVpc +} + +func (self *SWire) GetId() string { + return fmt.Sprintf("%s-%s", self.vpc.GetId(), self.region.GetId()) +} + +func (self *SWire) GetName() string { + return self.GetId() +} + +func (self *SWire) GetGlobalId() string { + return fmt.Sprintf("%s-%s", self.vpc.GetGlobalId(), self.region.GetGlobalId()) +} + +func (self *SWire) GetStatus() string { + return api.WIRE_STATUS_AVAILABLE +} + +func (self *SWire) GetIVpc() cloudprovider.ICloudVpc { + return self.vpc +} + +func (self *SWire) GetIZone() cloudprovider.ICloudZone { + return nil +} + +func (self *SWire) GetINetworks() ([]cloudprovider.ICloudNetwork, error) { + nets, err := self.region.GetNetwroks(self.vpc.Id) + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudNetwork{} + for i := range nets { + nets[i].wire = self + ret = append(ret, &nets[i]) + } + return ret, nil +} + +func (self *SWire) GetBandwidth() int { + return 10000 +} + +func (self *SWire) GetINetworkById(id string) (cloudprovider.ICloudNetwork, error) { + net, err := self.region.GetNetwork(id) + if err != nil { + return nil, err + } + net.wire = self + return net, nil +} + +/* +华为云子网可用区，类似一个zone标签。即使指定了zone子网在整个region依然是可用。 +通过华为web控制台创建子网需要指定可用区。这里是不指定的。 +*/ +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + net, err := self.region.CreateNetwork(self.vpc.GetId(), opts.Name, opts.Cidr, opts.Desc) + if err != nil { + return nil, errors.Wrapf(err, "CreateNetwork") + } + net.wire = self + return net, nil +} + +func getDefaultGateWay(cidr string) (string, error) { + pref, err := netutils.NewIPV4Prefix(cidr) + if err != nil { + return "", errors.Wrap(err, "getDefaultGateWay.NewIPV4Prefix") + } + startIp := pref.Address.NetAddr(pref.MaskLen) // 0 + startIp = startIp.StepUp() // 1 + return startIp.String(), nil +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090590.html +// cidr 掩码长度不能大于28 +func (self *SRegion) CreateNetwork(vpcId string, name string, cidr string, desc string) (*SNetwork, error) { + gateway, err := getDefaultGateWay(cidr) + if err != nil { + return nil, err + } + + params := map[string]interface{}{ + "subnet": map[string]interface{}{ + "name": name, + "vpc_id": vpcId, + "cidr": cidr, + "gateway_ip": gateway, + }, + } + ret := &SNetwork{} + return ret, self.vpcCreate("subnets", params, ret) +} diff --git a/pkg/multicloud/hcs/zone.go b/pkg/multicloud/hcs/zone.go new file mode 100644 index 0000000000..7ea1dd01a3 --- /dev/null +++ b/pkg/multicloud/hcs/zone.go @@ -0,0 +1,230 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hcs + +import ( + "fmt" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SZone struct { + multicloud.SResourceBase + multicloud.HcsTags + + region *SRegion + ZoneName string + ZoneState struct { + Available bool + } +} + +func (self *SRegion) GetZones() ([]SZone, error) { + zones := []SZone{} + return zones, self.ecsList("os-availability-zone", nil, &zones) +} + +func (self *SZone) getStorageType() ([]string, error) { + return self.region.GetZoneSupportedDiskTypes(self.GetId()) +} + +func (self *SRegion) GetZoneSupportedDiskTypes(zoneId string) ([]string, error) { + dts, err := self.GetDiskTypes() + if err != nil { + return nil, errors.Wrap(err, "GetDiskTypes") + } + + ret := []string{} + for i := range dts { + if dts[i].IsAvaliableInZone(zoneId) { + ret = append(ret, dts[i].Name) + } + } + return ret, nil +} + +func (self *SZone) GetId() string { + return self.ZoneName +} + +func (self *SZone) GetName() string { + return fmt.Sprintf("%s %s", CLOUD_PROVIDER_HCS_CN, self.ZoneName) +} + +func (self *SZone) GetI18n() cloudprovider.SModelI18nTable { + en := fmt.Sprintf("%s %s", CLOUD_PROVIDER_HCS_EN, self.ZoneName) + table := cloudprovider.SModelI18nTable{} + table["name"] = cloudprovider.NewSModelI18nEntry(self.GetName()).CN(self.GetName()).EN(en) + return table +} + +func (self *SZone) GetGlobalId() string { + return fmt.Sprintf("%s/%s", self.region.GetGlobalId(), self.ZoneName) +} + +func (self *SZone) GetStatus() string { + return api.ZONE_ENABLE +} + +func (self *SZone) GetIRegion() cloudprovider.ICloudRegion { + return self.region +} + +func (self *SZone) GetIHosts() ([]cloudprovider.ICloudHost, error) { + host := &SHost{zone: self} + return []cloudprovider.ICloudHost{host}, nil +} + +func (self *SZone) GetIHostById(id string) (cloudprovider.ICloudHost, error) { + host := &SHost{zone: self} + if host.GetGlobalId() == id { + return host, nil + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SZone) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + storages, err := self.getStorageType() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudStorage{} + for i := range storages { + ret = append(ret, &SStorage{zone: self, storageType: storages[i]}) + } + return ret, nil +} + +func (self *SZone) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + storages, err := self.GetIStorages() + if err != nil { + return nil, err + } + for i := 0; i < len(storages); i += 1 { + if storages[i].GetGlobalId() == id { + return storages[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SZone) GetIWires() ([]cloudprovider.ICloudWire, error) { + vpcs, err := self.region.GetVpcs() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudWire{} + for i := range vpcs { + wire := &SWire{region: self.region, vpc: &vpcs[i]} + ret = append(ret, wire) + } + return ret, nil +} + +func (self *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) { + zones, err := self.GetZones() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudZone{} + for i := 0; i < len(zones); i++ { + zones[i].region = self + ret = append(ret, &zones[i]) + } + if len(zones) == 0 { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "no availabe zone found") + } + return ret, nil +} + +func (self *SRegion) GetIZoneById(id string) (cloudprovider.ICloudZone, error) { + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + if izones[i].GetGlobalId() == id { + return izones[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SRegion) GetIHostById(id string) (cloudprovider.ICloudHost, error) { + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + ihost, err := izones[i].GetIHostById(id) + if err == nil { + return ihost, nil + } else if errors.Cause(err) != cloudprovider.ErrNotFound { + return nil, err + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SRegion) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + iStores := make([]cloudprovider.ICloudStorage, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneStores, err := izones[i].GetIStorages() + if err != nil { + return nil, err + } + iStores = append(iStores, iZoneStores...) + } + return iStores, nil +} + +func (self *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + istore, err := izones[i].GetIStorageById(id) + if err == nil { + return istore, nil + } else if errors.Cause(err) != cloudprovider.ErrNotFound { + return nil, err + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SRegion) GetZoneById(id string) (*SZone, error) { + zones, err := self.GetZones() + if err != nil { + return nil, err + } + for i := 0; i < len(zones); i += 1 { + zones[i].region = self + if zones[i].GetId() == id || zones[i].GetGlobalId() == id { + return &zones[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, id) +} diff --git a/pkg/multicloud/hcso/elasticcache_instance.go b/pkg/multicloud/hcso/elasticcache_instance.go index e83fa4653b..6db08b78df 100644 --- a/pkg/multicloud/hcso/elasticcache_instance.go +++ b/pkg/multicloud/hcso/elasticcache_instance.go @@ -476,17 +476,17 @@ func (self *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheIn params.Set("maintain_end", jsonutils.NewString(ec.MaintainEnd)) } - if strings.ToLower(ec.ChargeType) == billing_api.BILLING_TYPE_PREPAID && ec.BC != nil { + if strings.ToLower(ec.ChargeType) == billing_api.BILLING_TYPE_PREPAID && ec.BillingCycle != nil { bssParam := jsonutils.NewDict() bssParam.Set("charging_mode", jsonutils.NewString("prePaid")) bssParam.Set("is_auto_pay", jsonutils.NewString("true")) - bssParam.Set("is_auto_renew", jsonutils.NewString(fmt.Sprintf("%v", ec.BC.AutoRenew))) - if ec.BC.GetMonths() >= 1 && ec.BC.GetMonths() >= 9 { + bssParam.Set("is_auto_renew", jsonutils.NewString(fmt.Sprintf("%v", ec.BillingCycle.AutoRenew))) + if ec.BillingCycle.GetMonths() >= 1 && ec.BillingCycle.GetMonths() >= 9 { bssParam.Set("period_type", jsonutils.NewString("month")) - bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BC.GetMonths()))) - } else if ec.BC.GetYears() >= 1 && ec.BC.GetYears() <= 3 { + bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BillingCycle.GetMonths()))) + } else if ec.BillingCycle.GetYears() >= 1 && ec.BillingCycle.GetYears() <= 3 { bssParam.Set("period_type", jsonutils.NewString("year")) - bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BC.GetYears()))) + bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BillingCycle.GetYears()))) } else { return nil, fmt.Errorf("region.CreateIElasticcaches invalid billing cycle.reqired month (1~9) or year(1~3)") } diff --git a/pkg/multicloud/huawei/elasticcache_instance.go b/pkg/multicloud/huawei/elasticcache_instance.go index 606989a7a9..36bf3835cc 100644 --- a/pkg/multicloud/huawei/elasticcache_instance.go +++ b/pkg/multicloud/huawei/elasticcache_instance.go @@ -485,17 +485,17 @@ func (self *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheIn params.Set("maintain_end", jsonutils.NewString(ec.MaintainEnd)) } - if strings.ToLower(ec.ChargeType) == billing_api.BILLING_TYPE_PREPAID && ec.BC != nil { + if ec.BillingCycle != nil { bssParam := jsonutils.NewDict() bssParam.Set("charging_mode", jsonutils.NewString("prePaid")) bssParam.Set("is_auto_pay", jsonutils.NewString("true")) - bssParam.Set("is_auto_renew", jsonutils.NewString(fmt.Sprintf("%v", ec.BC.AutoRenew))) - if ec.BC.GetMonths() >= 1 && ec.BC.GetMonths() >= 9 { + bssParam.Set("is_auto_renew", jsonutils.NewString(fmt.Sprintf("%v", ec.BillingCycle.AutoRenew))) + if ec.BillingCycle.GetMonths() >= 1 && ec.BillingCycle.GetMonths() >= 9 { bssParam.Set("period_type", jsonutils.NewString("month")) - bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BC.GetMonths()))) - } else if ec.BC.GetYears() >= 1 && ec.BC.GetYears() <= 3 { + bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BillingCycle.GetMonths()))) + } else if ec.BillingCycle.GetYears() >= 1 && ec.BillingCycle.GetYears() <= 3 { bssParam.Set("period_type", jsonutils.NewString("year")) - bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BC.GetYears()))) + bssParam.Set("period_num", jsonutils.NewInt(int64(ec.BillingCycle.GetYears()))) } else { return nil, fmt.Errorf("region.CreateIElasticcaches invalid billing cycle.reqired month (1~9) or year(1~3)") } diff --git a/pkg/multicloud/loader/loader.go b/pkg/multicloud/loader/loader.go index a6efd9204e..fbcdbcb11c 100644 --- a/pkg/multicloud/loader/loader.go +++ b/pkg/multicloud/loader/loader.go @@ -27,6 +27,7 @@ import ( _ "yunion.io/x/onecloud/pkg/multicloud/ecloud/provider" // public clouds _ "yunion.io/x/onecloud/pkg/multicloud/esxi/provider" // private clouds _ "yunion.io/x/onecloud/pkg/multicloud/google/provider" // public clouds + _ "yunion.io/x/onecloud/pkg/multicloud/hcs/provider" _ "yunion.io/x/onecloud/pkg/multicloud/hcso/provider" _ "yunion.io/x/onecloud/pkg/multicloud/huawei/provider" _ "yunion.io/x/onecloud/pkg/multicloud/incloudsphere/provider" // private clouds diff --git a/pkg/multicloud/qcloud/region.go b/pkg/multicloud/qcloud/region.go index 2fa042e08b..e611692cb3 100644 --- a/pkg/multicloud/qcloud/region.go +++ b/pkg/multicloud/qcloud/region.go @@ -1159,7 +1159,6 @@ func (r *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheInput params["InstanceName"] = ec.InstanceName if len(ec.ProjectId) > 0 { - log.Debugf("CreateIElasticcaches %s at project %s", ec.InstanceName, ec.ProjectId) params["ProjectId"] = ec.ProjectId } params["ZoneId"] = fmt.Sprintf("%d", zoneId) @@ -1168,7 +1167,7 @@ func (r *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheInput params["RedisShardNum"] = spec.RedisShardNum params["RedisReplicasNum"] = spec.RedisReplicasNum params["GoodsNum"] = "1" - if strings.ToLower(ec.NetworkType) == api.LB_NETWORK_TYPE_VPC { + if ec.NetworkType == api.LB_NETWORK_TYPE_VPC { params["VpcId"] = ec.VpcId params["SubnetId"] = ec.NetworkId @@ -1178,11 +1177,11 @@ func (r *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheInput } params["Period"] = "1" params["BillingMode"] = "0" - if ec.BC != nil && ec.BC.GetMonths() >= 1 { - params["Period"] = strconv.Itoa(ec.BC.GetMonths()) + if ec.BillingCycle != nil && ec.BillingCycle.GetMonths() >= 1 { + params["Period"] = strconv.Itoa(ec.BillingCycle.GetMonths()) params["BillingMode"] = "1" // 自动续费 - if ec.BC.AutoRenew { + if ec.BillingCycle.AutoRenew { params["AutoRenew"] = "1" } } diff --git a/pkg/multicloud/region_base.go b/pkg/multicloud/region_base.go index 16b6873fa1..790049e27e 100644 --- a/pkg/multicloud/region_base.go +++ b/pkg/multicloud/region_base.go @@ -127,6 +127,10 @@ func (self *SRegion) GetIElasticcaches() ([]cloudprovider.ICloudElasticcache, er return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetIElasticcaches") } +func (self *SRegion) GetIElasticcacheSkus() ([]cloudprovider.ICloudElasticcacheSku, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetIElasticcacheSkus") +} + func (self *SRegion) CreateIDBInstance(desc *cloudprovider.SManagedDBInstanceCreateConfig) (cloudprovider.ICloudDBInstance, error) { return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "CreateIDBInstance") } @@ -355,7 +359,7 @@ func (self *SRegionSecurityGroupBase) CreateISecurityGroup(conf *cloudprovider.S } func (self *SRegionSecurityGroupBase) GetISecurityGroupById(secgroupId string) (cloudprovider.ICloudSecurityGroup, error) { - return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetIElasticSearchById") + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISecurityGroupById") } func (self *SRegionSecurityGroupBase) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { diff --git a/pkg/multicloud/tag_base.go b/pkg/multicloud/tag_base.go index b91a897c63..2c05319e46 100644 --- a/pkg/multicloud/tag_base.go +++ b/pkg/multicloud/tag_base.go @@ -534,3 +534,26 @@ func (self *RemoteFileTags) GetSysTags() map[string]string { func (self *RemoteFileTags) SetTags(tags map[string]string, replace bool) error { return errors.Wrap(cloudprovider.ErrNotImplemented, "SetTags") } + +type HcsTags struct { + Tags []string +} + +func (self *HcsTags) GetTags() (map[string]string, error) { + tags := map[string]string{} + for _, kv := range self.Tags { + splited := strings.Split(kv, "=") + if len(splited) == 2 { + tags[splited[0]] = splited[1] + } + } + return tags, nil +} + +func (self *HcsTags) GetSysTags() map[string]string { + return nil +} + +func (self *HcsTags) SetTags(tags map[string]string, replace bool) error { + return errors.Wrap(cloudprovider.ErrNotImplemented, "SetTags") +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/LICENSE b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/LICENSE new file mode 100644 index 0000000000..4cd54f94ec --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 Huawei Technologies Co., Ltd. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/auth.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/auth.go new file mode 100644 index 0000000000..00c74cf425 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/auth.go @@ -0,0 +1,320 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "fmt" + "net/url" + "sort" + "strings" + "time" +) + +func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, expires int64) (requestURL string, err error) { + sh := obsClient.getSecurity() + isAkSkEmpty := sh.ak == "" || sh.sk == "" + if isAkSkEmpty == false && sh.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = sh.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = sh.securityToken + } + } + requestURL, canonicalizedURL := obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestURL, err := url.Parse(requestURL) + if err != nil { + return "", err + } + encodeHeaders(headers) + hostName := parsedRequestURL.Host + + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) + + if isAkSkEmpty { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { + if isV4 { + date, parseDateErr := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } + delete(headers, HEADER_DATE_CAMEL) + shortDate := date.Format(SHORT_DATE_FORMAT) + longDate := date.Format(LONG_DATE_FORMAT) + if len(headers[HEADER_HOST_CAMEL]) != 0 { + index := strings.LastIndex(headers[HEADER_HOST_CAMEL][0], ":") + if index != -1 { + port := headers[HEADER_HOST_CAMEL][0][index+1:] + if port == "80" || port == "443" { + headers[HEADER_HOST_CAMEL] = []string{headers[HEADER_HOST_CAMEL][0][:index]} + } + } + + } + + signedHeaders, _headers := getSignedHeaders(headers) + + credential, scope := getCredential(sh.ak, obsClient.conf.region, shortDate) + params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX + params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + params[PARAM_DATE_AMZ_CAMEL] = longDate + params[PARAM_EXPIRES_AMZ_CAMEL] = Int64ToString(expires) + params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = strings.Join(signedHeaders, ";") + + requestURL, canonicalizedURL = obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestURL, _err := url.Parse(requestURL) + if _err != nil { + return "", _err + } + + stringToSign := getV4StringToSign(method, canonicalizedURL, parsedRequestURL.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, signedHeaders, _headers) + signature := getSignature(stringToSign, sh.sk, obsClient.conf.region, shortDate) + + requestURL += fmt.Sprintf("&%s=%s", PARAM_SIGNATURE_AMZ_CAMEL, UrlEncode(signature, false)) + + } else { + originDate := headers[HEADER_DATE_CAMEL][0] + date, parseDateErr := time.Parse(RFC1123_FORMAT, originDate) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } + expires += date.Unix() + headers[HEADER_DATE_CAMEL] = []string{Int64ToString(expires)} + + stringToSign := getV2StringToSign(method, canonicalizedURL, headers, obsClient.conf.signature == SignatureObs) + signature := UrlEncode(Base64Encode(HmacSha1([]byte(sh.sk), []byte(stringToSign))), false) + if strings.Index(requestURL, "?") < 0 { + requestURL += "?" + } else { + requestURL += "&" + } + delete(headers, HEADER_DATE_CAMEL) + + if obsClient.conf.signature != SignatureObs { + requestURL += "AWS" + } + requestURL += fmt.Sprintf("AccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(sh.ak, false), expires, signature) + } + } + + return +} + +func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, hostName string) (requestURL string, err error) { + sh := obsClient.getSecurity() + isAkSkEmpty := sh.ak == "" || sh.sk == "" + if isAkSkEmpty == false && sh.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + headers[HEADER_STS_TOKEN_OBS] = []string{sh.securityToken} + } else { + headers[HEADER_STS_TOKEN_AMZ] = []string{sh.securityToken} + } + } + isObs := obsClient.conf.signature == SignatureObs + requestURL, canonicalizedURL := obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestURL, err := url.Parse(requestURL) + if err != nil { + return "", err + } + encodeHeaders(headers) + + if hostName == "" { + hostName = parsedRequestURL.Host + } + + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) + + if isAkSkEmpty { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { + ak := sh.ak + sk := sh.sk + var authorization string + if isV4 { + headers[HEADER_CONTENT_SHA256_AMZ] = []string{UNSIGNED_PAYLOAD} + ret := v4Auth(ak, sk, obsClient.conf.region, method, canonicalizedURL, parsedRequestURL.RawQuery, headers) + authorization = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else { + ret := v2Auth(ak, sk, method, canonicalizedURL, headers, isObs) + hashPrefix := V2_HASH_PREFIX + if isObs { + hashPrefix = OBS_HASH_PREFIX + } + authorization = fmt.Sprintf("%s %s:%s", hashPrefix, ak, ret["Signature"]) + } + headers[HEADER_AUTH_CAMEL] = []string{authorization} + } + return +} + +func prepareHostAndDate(headers map[string][]string, hostName string, isV4 bool) { + headers[HEADER_HOST_CAMEL] = []string{hostName} + if date, ok := headers[HEADER_DATE_AMZ]; ok { + flag := false + if len(date) == 1 { + if isV4 { + if t, err := time.Parse(LONG_DATE_FORMAT, date[0]); err == nil { + headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(t)} + flag = true + } + } else { + if strings.HasSuffix(date[0], "GMT") { + headers[HEADER_DATE_CAMEL] = []string{date[0]} + flag = true + } + } + } + if !flag { + delete(headers, HEADER_DATE_AMZ) + } + } + if _, ok := headers[HEADER_DATE_CAMEL]; !ok { + headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(time.Now().UTC())} + } + +} + +func encodeHeaders(headers map[string][]string) { + for key, values := range headers { + for index, value := range values { + values[index] = UrlEncode(value, true) + } + headers[key] = values + } +} + +func prepareDateHeader(dataHeader, dateCamelHeader string, headers, _headers map[string][]string) { + if _, ok := _headers[HEADER_DATE_CAMEL]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } else if _, ok := _headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } +} + +func getStringToSign(keys []string, isObs bool, _headers map[string][]string) []string { + stringToSign := make([]string, 0, len(keys)) + for _, key := range keys { + var value string + prefixHeader := HEADER_PREFIX + prefixMetaHeader := HEADER_PREFIX_META + if isObs { + prefixHeader = HEADER_PREFIX_OBS + prefixMetaHeader = HEADER_PREFIX_META_OBS + } + if strings.HasPrefix(key, prefixHeader) { + if strings.HasPrefix(key, prefixMetaHeader) { + for index, v := range _headers[key] { + value += strings.TrimSpace(v) + if index != len(_headers[key])-1 { + value += "," + } + } + } else { + value = strings.Join(_headers[key], ",") + } + value = fmt.Sprintf("%s:%s", key, value) + } else { + value = strings.Join(_headers[key], ",") + } + stringToSign = append(stringToSign, value) + } + return stringToSign +} + +func attachHeaders(headers map[string][]string, isObs bool) string { + length := len(headers) + _headers := make(map[string][]string, length) + keys := make([]string, 0, length) + + for key, value := range headers { + _key := strings.ToLower(strings.TrimSpace(key)) + if _key != "" { + prefixheader := HEADER_PREFIX + if isObs { + prefixheader = HEADER_PREFIX_OBS + } + if _key == "content-md5" || _key == "content-type" || _key == "date" || strings.HasPrefix(_key, prefixheader) { + keys = append(keys, _key) + _headers[_key] = value + } + } else { + delete(headers, key) + } + } + + for _, interestedHeader := range interestedHeaders { + if _, ok := _headers[interestedHeader]; !ok { + _headers[interestedHeader] = []string{""} + keys = append(keys, interestedHeader) + } + } + dateCamelHeader := PARAM_DATE_AMZ_CAMEL + dataHeader := HEADER_DATE_AMZ + if isObs { + dateCamelHeader = PARAM_DATE_OBS_CAMEL + dataHeader = HEADER_DATE_OBS + } + prepareDateHeader(dataHeader, dateCamelHeader, headers, _headers) + + sort.Strings(keys) + stringToSign := getStringToSign(keys, isObs, _headers) + return strings.Join(stringToSign, "\n") +} + +func getScope(region, shortDate string) string { + return fmt.Sprintf("%s/%s/%s/%s", shortDate, region, V4_SERVICE_NAME, V4_SERVICE_SUFFIX) +} + +func getCredential(ak, region, shortDate string) (string, string) { + scope := getScope(region, shortDate) + return fmt.Sprintf("%s/%s", ak, scope), scope +} + +func getSignedHeaders(headers map[string][]string) ([]string, map[string][]string) { + length := len(headers) + _headers := make(map[string][]string, length) + signedHeaders := make([]string, 0, length) + for key, value := range headers { + _key := strings.ToLower(strings.TrimSpace(key)) + if _key != "" { + signedHeaders = append(signedHeaders, _key) + _headers[_key] = value + } else { + delete(headers, key) + } + } + sort.Strings(signedHeaders) + return signedHeaders, _headers +} + +func getSignature(stringToSign, sk, region, shortDate string) string { + key := HmacSha256([]byte(V4_HASH_PRE+sk), []byte(shortDate)) + key = HmacSha256(key, []byte(region)) + key = HmacSha256(key, []byte(V4_SERVICE_NAME)) + key = HmacSha256(key, []byte(V4_SERVICE_SUFFIX)) + return Hex(HmacSha256(key, []byte(stringToSign))) +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/authV2.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/authV2.go new file mode 100644 index 0000000000..1812061f02 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/authV2.go @@ -0,0 +1,55 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "strings" +) + +func getV2StringToSign(method, canonicalizedURL string, headers map[string][]string, isObs bool) string { + stringToSign := strings.Join([]string{method, "\n", attachHeaders(headers, isObs), "\n", canonicalizedURL}, "") + + var isSecurityToken bool + var securityToken []string + if isObs { + securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_OBS] + } else { + securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_AMZ] + } + var query []string + if !isSecurityToken { + parmas := strings.Split(canonicalizedURL, "?") + if len(parmas) > 1 { + query = strings.Split(parmas[1], "&") + for _, value := range query { + if strings.HasPrefix(value, HEADER_STS_TOKEN_AMZ+"=") || strings.HasPrefix(value, HEADER_STS_TOKEN_OBS+"=") { + if value[len(HEADER_STS_TOKEN_AMZ)+1:] != "" { + securityToken = []string{value[len(HEADER_STS_TOKEN_AMZ)+1:]} + isSecurityToken = true + } + } + } + } + } + logStringToSign := stringToSign + if isSecurityToken && len(securityToken) > 0 { + logStringToSign = strings.Replace(logStringToSign, securityToken[0], "******", -1) + } + doLog(LEVEL_DEBUG, "The v2 auth stringToSign:\n%s", logStringToSign) + return stringToSign +} + +func v2Auth(ak, sk, method, canonicalizedURL string, headers map[string][]string, isObs bool) map[string]string { + stringToSign := getV2StringToSign(method, canonicalizedURL, headers, isObs) + return map[string]string{"Signature": Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign)))} +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/authV4.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/authV4.go new file mode 100644 index 0000000000..70abe96ecc --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/authV4.go @@ -0,0 +1,137 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "strings" + "time" +) + +func getV4StringToSign(method, canonicalizedURL, queryURL, scope, longDate, payload string, signedHeaders []string, headers map[string][]string) string { + canonicalRequest := make([]string, 0, 10+len(signedHeaders)*4) + canonicalRequest = append(canonicalRequest, method) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, canonicalizedURL) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, queryURL) + canonicalRequest = append(canonicalRequest, "\n") + + for _, signedHeader := range signedHeaders { + values, _ := headers[signedHeader] + for _, value := range values { + canonicalRequest = append(canonicalRequest, signedHeader) + canonicalRequest = append(canonicalRequest, ":") + canonicalRequest = append(canonicalRequest, value) + canonicalRequest = append(canonicalRequest, "\n") + } + } + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, strings.Join(signedHeaders, ";")) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, payload) + + _canonicalRequest := strings.Join(canonicalRequest, "") + + var isSecurityToken bool + var securityToken []string + if securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_OBS]; !isSecurityToken { + securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_AMZ] + } + var query []string + if !isSecurityToken { + query = strings.Split(queryURL, "&") + for _, value := range query { + if strings.HasPrefix(value, HEADER_STS_TOKEN_AMZ+"=") || strings.HasPrefix(value, HEADER_STS_TOKEN_OBS+"=") { + if value[len(HEADER_STS_TOKEN_AMZ)+1:] != "" { + securityToken = []string{value[len(HEADER_STS_TOKEN_AMZ)+1:]} + isSecurityToken = true + } + } + } + } + logCanonicalRequest := _canonicalRequest + if isSecurityToken && len(securityToken) > 0 { + logCanonicalRequest = strings.Replace(logCanonicalRequest, securityToken[0], "******", -1) + } + doLog(LEVEL_DEBUG, "The v4 auth canonicalRequest:\n%s", logCanonicalRequest) + + stringToSign := make([]string, 0, 7) + stringToSign = append(stringToSign, V4_HASH_PREFIX) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, longDate) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, scope) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, HexSha256([]byte(_canonicalRequest))) + + _stringToSign := strings.Join(stringToSign, "") + + doLog(LEVEL_DEBUG, "The v4 auth stringToSign:\n%s", _stringToSign) + return _stringToSign +} + +// V4Auth is a wrapper for v4Auth +func V4Auth(ak, sk, region, method, canonicalizedURL, queryURL string, headers map[string][]string) map[string]string { + return v4Auth(ak, sk, region, method, canonicalizedURL, queryURL, headers) +} + +func v4Auth(ak, sk, region, method, canonicalizedURL, queryURL string, headers map[string][]string) map[string]string { + var t time.Time + if val, ok := headers[HEADER_DATE_AMZ]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[PARAM_DATE_AMZ_CAMEL]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[HEADER_DATE_CAMEL]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else { + t = time.Now().UTC() + } + shortDate := t.Format(SHORT_DATE_FORMAT) + longDate := t.Format(LONG_DATE_FORMAT) + + signedHeaders, _headers := getSignedHeaders(headers) + + credential, scope := getCredential(ak, region, shortDate) + + payload := UNSIGNED_PAYLOAD + if val, ok := headers[HEADER_CONTENT_SHA256_AMZ]; ok { + payload = val[0] + } + stringToSign := getV4StringToSign(method, canonicalizedURL, queryURL, scope, longDate, payload, signedHeaders, _headers) + + signature := getSignature(stringToSign, sk, region, shortDate) + + ret := make(map[string]string, 3) + ret["Credential"] = credential + ret["SignedHeaders"] = strings.Join(signedHeaders, ";") + ret["Signature"] = signature + return ret +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_base.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_base.go new file mode 100644 index 0000000000..501ab5e0d4 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_base.go @@ -0,0 +1,61 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "fmt" + "net/http" + "strings" +) + +// ObsClient defines OBS client. +type ObsClient struct { + conf *config + httpClient *http.Client +} + +// New creates a new ObsClient instance. +func New(ak, sk, endpoint string, configurers ...configurer) (*ObsClient, error) { + conf := &config{endpoint: endpoint} + conf.securityProviders = make([]securityProvider, 0, 3) + conf.securityProviders = append(conf.securityProviders, NewBasicSecurityProvider(ak, sk, "")) + + conf.maxRetryCount = -1 + conf.maxRedirectCount = -1 + for _, configurer := range configurers { + configurer(conf) + } + + if err := conf.initConfigWithDefault(); err != nil { + return nil, err + } + err := conf.getTransport() + if err != nil { + return nil, err + } + + if isWarnLogEnabled() { + info := make([]string, 3) + info[0] = fmt.Sprintf("[OBS SDK Version=%s", obsSdkVersion) + info[1] = fmt.Sprintf("Endpoint=%s", conf.endpoint) + accessMode := "Virtual Hosting" + if conf.pathStyle { + accessMode = "Path" + } + info[2] = fmt.Sprintf("Access Mode=%s]", accessMode) + doLog(LEVEL_WARN, strings.Join(info, "];[")) + } + doLog(LEVEL_DEBUG, "Create obsclient with config:\n%s\n", conf) + obsClient := &ObsClient{conf: conf, httpClient: &http.Client{Transport: conf.transport, CheckRedirect: checkRedirectFunc}} + return obsClient, nil +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_bucket.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_bucket.go new file mode 100644 index 0000000000..f8b2468f3b --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_bucket.go @@ -0,0 +1,731 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" + "fmt" + "strings" +) + +// ListBuckets lists buckets. +// +// You can use this API to obtain the bucket list. In the list, bucket names are displayed in lexicographical order. +func (obsClient ObsClient) ListBuckets(input *ListBucketsInput, extensions ...extensionOptions) (output *ListBucketsOutput, err error) { + if input == nil { + input = &ListBucketsInput{} + } + output = &ListBucketsOutput{} + err = obsClient.doActionWithoutBucket("ListBuckets", HTTP_GET, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// CreateBucket creates a bucket. +// +// You can use this API to create a bucket and name it as you specify. The created bucket name must be unique in OBS. +func (obsClient ObsClient) CreateBucket(input *CreateBucketInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("CreateBucketInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("CreateBucket", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucket deletes a bucket. +// +// You can use this API to delete a bucket. The bucket to be deleted must be empty +// (containing no objects, noncurrent object versions, or part fragments). +func (obsClient ObsClient) DeleteBucket(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucket", HTTP_DELETE, bucketName, defaultSerializable, output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketStoragePolicy sets bucket storage class. +// +// You can use this API to set storage class for bucket. +func (obsClient ObsClient) SetBucketStoragePolicy(input *SetBucketStoragePolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketStoragePolicyInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketStoragePolicy", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} +func (obsClient ObsClient) getBucketStoragePolicyS3(bucketName string, extensions []extensionOptions) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + var outputS3 *getBucketStoragePolicyOutputS3 + outputS3 = &getBucketStoragePolicyOutputS3{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStoragePolicy), outputS3, extensions) + if err != nil { + output = nil + return + } + output.BaseModel = outputS3.BaseModel + output.StorageClass = fmt.Sprintf("%s", outputS3.StorageClass) + return +} + +func (obsClient ObsClient) getBucketStoragePolicyObs(bucketName string, extensions []extensionOptions) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + var outputObs *getBucketStoragePolicyOutputObs + outputObs = &getBucketStoragePolicyOutputObs{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageClass), outputObs, extensions) + if err != nil { + output = nil + return + } + output.BaseModel = outputObs.BaseModel + output.StorageClass = outputObs.StorageClass + return +} + +// GetBucketStoragePolicy gets bucket storage class. +// +// You can use this API to obtain the storage class of a bucket. +func (obsClient ObsClient) GetBucketStoragePolicy(bucketName string, extensions ...extensionOptions) (output *GetBucketStoragePolicyOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketStoragePolicyObs(bucketName, extensions) + } + return obsClient.getBucketStoragePolicyS3(bucketName, extensions) +} + +// SetBucketQuota sets the bucket quota. +// +// You can use this API to set the bucket quota. A bucket quota must be expressed in bytes and the maximum value is 2^63-1. +func (obsClient ObsClient) SetBucketQuota(input *SetBucketQuotaInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketQuotaInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketQuota", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketQuota gets the bucket quota. +// +// You can use this API to obtain the bucket quota. Value 0 indicates that no upper limit is set for the bucket quota. +func (obsClient ObsClient) GetBucketQuota(bucketName string, extensions ...extensionOptions) (output *GetBucketQuotaOutput, err error) { + output = &GetBucketQuotaOutput{} + err = obsClient.doActionWithBucket("GetBucketQuota", HTTP_GET, bucketName, newSubResourceSerial(SubResourceQuota), output, extensions) + if err != nil { + output = nil + } + return +} + +// HeadBucket checks whether a bucket exists. +// +// You can use this API to check whether a bucket exists. +func (obsClient ObsClient) HeadBucket(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("HeadBucket", HTTP_HEAD, bucketName, defaultSerializable, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketMetadata gets the metadata of a bucket. +// +// You can use this API to send a HEAD request to a bucket to obtain the bucket +// metadata such as the storage class and CORS rules (if set). +func (obsClient ObsClient) GetBucketMetadata(input *GetBucketMetadataInput, extensions ...extensionOptions) (output *GetBucketMetadataOutput, err error) { + output = &GetBucketMetadataOutput{} + err = obsClient.doActionWithBucket("GetBucketMetadata", HTTP_HEAD, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else { + ParseGetBucketMetadataOutput(output) + } + return +} + +// GetBucketStorageInfo gets storage information about a bucket. +// +// You can use this API to obtain storage information about a bucket, including the +// bucket size and number of objects in the bucket. +func (obsClient ObsClient) GetBucketStorageInfo(bucketName string, extensions ...extensionOptions) (output *GetBucketStorageInfoOutput, err error) { + output = &GetBucketStorageInfoOutput{} + err = obsClient.doActionWithBucket("GetBucketStorageInfo", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageInfo), output, extensions) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) getBucketLocationS3(bucketName string, extensions []extensionOptions) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + var outputS3 *getBucketLocationOutputS3 + outputS3 = &getBucketLocationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputS3, extensions) + if err != nil { + output = nil + } else { + output.BaseModel = outputS3.BaseModel + output.Location = outputS3.Location + } + return +} +func (obsClient ObsClient) getBucketLocationObs(bucketName string, extensions []extensionOptions) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + var outputObs *getBucketLocationOutputObs + outputObs = &getBucketLocationOutputObs{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputObs, extensions) + if err != nil { + output = nil + } else { + output.BaseModel = outputObs.BaseModel + output.Location = outputObs.Location + } + return +} + +// GetBucketLocation gets the location of a bucket. +// +// You can use this API to obtain the bucket location. +func (obsClient ObsClient) GetBucketLocation(bucketName string, extensions ...extensionOptions) (output *GetBucketLocationOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketLocationObs(bucketName, extensions) + } + return obsClient.getBucketLocationS3(bucketName, extensions) +} + +// SetBucketAcl sets the bucket ACL. +// +// You can use this API to set the ACL for a bucket. +func (obsClient ObsClient) SetBucketAcl(input *SetBucketAclInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketAclInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketAcl", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} +func (obsClient ObsClient) getBucketACLObs(bucketName string, extensions []extensionOptions) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + var outputObs *getBucketACLOutputObs + outputObs = &getBucketACLOutputObs{} + err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), outputObs, extensions) + if err != nil { + output = nil + } else { + output.BaseModel = outputObs.BaseModel + output.Owner = outputObs.Owner + output.Grants = make([]Grant, 0, len(outputObs.Grants)) + for _, valGrant := range outputObs.Grants { + tempOutput := Grant{} + tempOutput.Delivered = valGrant.Delivered + tempOutput.Permission = valGrant.Permission + tempOutput.Grantee.DisplayName = valGrant.Grantee.DisplayName + tempOutput.Grantee.ID = valGrant.Grantee.ID + tempOutput.Grantee.Type = valGrant.Grantee.Type + tempOutput.Grantee.URI = GroupAllUsers + + output.Grants = append(output.Grants, tempOutput) + } + } + return +} + +// GetBucketAcl gets the bucket ACL. +// +// You can use this API to obtain a bucket ACL. +func (obsClient ObsClient) GetBucketAcl(bucketName string, extensions ...extensionOptions) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketACLObs(bucketName, extensions) + } + err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketPolicy sets the bucket policy. +// +// You can use this API to set a bucket policy. If the bucket already has a policy, the +// policy will be overwritten by the one specified in this request. +func (obsClient ObsClient) SetBucketPolicy(input *SetBucketPolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketPolicy is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketPolicy", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketPolicy gets the bucket policy. +// +// You can use this API to obtain the policy of a bucket. +func (obsClient ObsClient) GetBucketPolicy(bucketName string, extensions ...extensionOptions) (output *GetBucketPolicyOutput, err error) { + output = &GetBucketPolicyOutput{} + err = obsClient.doActionWithBucketV2("GetBucketPolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourcePolicy), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketPolicy deletes the bucket policy. +// +// You can use this API to delete the policy of a bucket. +func (obsClient ObsClient) DeleteBucketPolicy(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketPolicy", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourcePolicy), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketCors sets CORS rules for a bucket. +// +// You can use this API to set CORS rules for a bucket to allow client browsers to send cross-origin requests. +func (obsClient ObsClient) SetBucketCors(input *SetBucketCorsInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketCorsInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketCors", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketCors gets CORS rules of a bucket. +// +// You can use this API to obtain the CORS rules of a specified bucket. +func (obsClient ObsClient) GetBucketCors(bucketName string, extensions ...extensionOptions) (output *GetBucketCorsOutput, err error) { + output = &GetBucketCorsOutput{} + err = obsClient.doActionWithBucket("GetBucketCors", HTTP_GET, bucketName, newSubResourceSerial(SubResourceCors), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketCors deletes CORS rules of a bucket. +// +// You can use this API to delete the CORS rules of a specified bucket. +func (obsClient ObsClient) DeleteBucketCors(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketCors", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceCors), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketVersioning sets the versioning status for a bucket. +// +// You can use this API to set the versioning status for a bucket. +func (obsClient ObsClient) SetBucketVersioning(input *SetBucketVersioningInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketVersioningInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketVersioning", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketVersioning gets the versioning status of a bucket. +// +// You can use this API to obtain the versioning status of a bucket. +func (obsClient ObsClient) GetBucketVersioning(bucketName string, extensions ...extensionOptions) (output *GetBucketVersioningOutput, err error) { + output = &GetBucketVersioningOutput{} + err = obsClient.doActionWithBucket("GetBucketVersioning", HTTP_GET, bucketName, newSubResourceSerial(SubResourceVersioning), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketWebsiteConfiguration sets website hosting for a bucket. +// +// You can use this API to set website hosting for a bucket. +func (obsClient ObsClient) SetBucketWebsiteConfiguration(input *SetBucketWebsiteConfigurationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketWebsiteConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketWebsiteConfiguration", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketWebsiteConfiguration gets the website hosting settings of a bucket. +// +// You can use this API to obtain the website hosting settings of a bucket. +func (obsClient ObsClient) GetBucketWebsiteConfiguration(bucketName string, extensions ...extensionOptions) (output *GetBucketWebsiteConfigurationOutput, err error) { + output = &GetBucketWebsiteConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketWebsiteConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceWebsite), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketWebsiteConfiguration deletes the website hosting settings of a bucket. +// +// You can use this API to delete the website hosting settings of a bucket. +func (obsClient ObsClient) DeleteBucketWebsiteConfiguration(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketWebsiteConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceWebsite), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketLoggingConfiguration sets the bucket logging. +// +// You can use this API to configure access logging for a bucket. +func (obsClient ObsClient) SetBucketLoggingConfiguration(input *SetBucketLoggingConfigurationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketLoggingConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketLoggingConfiguration", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketLoggingConfiguration gets the logging settings of a bucket. +// +// You can use this API to obtain the access logging settings of a bucket. +func (obsClient ObsClient) GetBucketLoggingConfiguration(bucketName string, extensions ...extensionOptions) (output *GetBucketLoggingConfigurationOutput, err error) { + output = &GetBucketLoggingConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketLoggingConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLogging), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketLifecycleConfiguration sets lifecycle rules for a bucket. +// +// You can use this API to set lifecycle rules for a bucket, to periodically transit +// storage classes of objects and delete objects in the bucket. +func (obsClient ObsClient) SetBucketLifecycleConfiguration(input *SetBucketLifecycleConfigurationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketLifecycleConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketLifecycleConfiguration", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketLifecycleConfiguration gets lifecycle rules of a bucket. +// +// You can use this API to obtain the lifecycle rules of a bucket. +func (obsClient ObsClient) GetBucketLifecycleConfiguration(bucketName string, extensions ...extensionOptions) (output *GetBucketLifecycleConfigurationOutput, err error) { + output = &GetBucketLifecycleConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketLifecycleConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLifecycle), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketLifecycleConfiguration deletes lifecycle rules of a bucket. +// +// You can use this API to delete all lifecycle rules of a bucket. +func (obsClient ObsClient) DeleteBucketLifecycleConfiguration(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketLifecycleConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceLifecycle), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketEncryption sets the default server-side encryption for a bucket. +// +// You can use this API to create or update the default server-side encryption for a bucket. +func (obsClient ObsClient) SetBucketEncryption(input *SetBucketEncryptionInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketEncryptionInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketEncryption", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketEncryption gets the encryption configuration of a bucket. +// +// You can use this API to obtain obtain the encryption configuration of a bucket. +func (obsClient ObsClient) GetBucketEncryption(bucketName string, extensions ...extensionOptions) (output *GetBucketEncryptionOutput, err error) { + output = &GetBucketEncryptionOutput{} + err = obsClient.doActionWithBucket("GetBucketEncryption", HTTP_GET, bucketName, newSubResourceSerial(SubResourceEncryption), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketEncryption deletes the encryption configuration of a bucket. +// +// You can use this API to delete the encryption configuration of a bucket. +func (obsClient ObsClient) DeleteBucketEncryption(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketEncryption", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceEncryption), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketTagging sets bucket tags. +// +// You can use this API to set bucket tags. +func (obsClient ObsClient) SetBucketTagging(input *SetBucketTaggingInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketTaggingInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketTagging", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketTagging gets bucket tags. +// +// You can use this API to obtain the tags of a specified bucket. +func (obsClient ObsClient) GetBucketTagging(bucketName string, extensions ...extensionOptions) (output *GetBucketTaggingOutput, err error) { + output = &GetBucketTaggingOutput{} + err = obsClient.doActionWithBucket("GetBucketTagging", HTTP_GET, bucketName, newSubResourceSerial(SubResourceTagging), output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketTagging deletes bucket tags. +// +// You can use this API to delete the tags of a specified bucket. +func (obsClient ObsClient) DeleteBucketTagging(bucketName string, extensions ...extensionOptions) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketTagging", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceTagging), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketNotification sets event notification for a bucket. +// +// You can use this API to configure event notification for a bucket. You will be notified of all +// specified operations performed on the bucket. +func (obsClient ObsClient) SetBucketNotification(input *SetBucketNotificationInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketNotificationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketNotification", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketNotification gets event notification settings of a bucket. +// +// You can use this API to obtain the event notification configuration of a bucket. +func (obsClient ObsClient) GetBucketNotification(bucketName string, extensions ...extensionOptions) (output *GetBucketNotificationOutput, err error) { + if obsClient.conf.signature != SignatureObs { + return obsClient.getBucketNotificationS3(bucketName, extensions) + } + output = &GetBucketNotificationOutput{} + err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), output, extensions) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) getBucketNotificationS3(bucketName string, extensions []extensionOptions) (output *GetBucketNotificationOutput, err error) { + outputS3 := &getBucketNotificationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), outputS3, extensions) + if err != nil { + return nil, err + } + + output = &GetBucketNotificationOutput{} + output.BaseModel = outputS3.BaseModel + topicConfigurations := make([]TopicConfiguration, 0, len(outputS3.TopicConfigurations)) + for _, topicConfigurationS3 := range outputS3.TopicConfigurations { + topicConfiguration := TopicConfiguration{} + topicConfiguration.ID = topicConfigurationS3.ID + topicConfiguration.Topic = topicConfigurationS3.Topic + topicConfiguration.FilterRules = topicConfigurationS3.FilterRules + + events := make([]EventType, 0, len(topicConfigurationS3.Events)) + for _, event := range topicConfigurationS3.Events { + events = append(events, ParseStringToEventType(event)) + } + topicConfiguration.Events = events + topicConfigurations = append(topicConfigurations, topicConfiguration) + } + output.TopicConfigurations = topicConfigurations + return +} + +// SetBucketRequestPayment sets requester-pays setting for a bucket. +func (obsClient ObsClient) SetBucketRequestPayment(input *SetBucketRequestPaymentInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketRequestPaymentInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketRequestPayment", HTTP_PUT, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketRequestPayment gets requester-pays setting of a bucket. +func (obsClient ObsClient) GetBucketRequestPayment(bucketName string, extensions ...extensionOptions) (output *GetBucketRequestPaymentOutput, err error) { + output = &GetBucketRequestPaymentOutput{} + err = obsClient.doActionWithBucket("GetBucketRequestPayment", HTTP_GET, bucketName, newSubResourceSerial(SubResourceRequestPayment), output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketFetchPolicy sets the bucket fetch policy. +// +// You can use this API to set a bucket fetch policy. +func (obsClient ObsClient) SetBucketFetchPolicy(input *SetBucketFetchPolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketFetchPolicyInput is nil") + } + if strings.TrimSpace(string(input.Status)) == "" { + return nil, errors.New("Fetch policy status is empty") + } + if strings.TrimSpace(input.Agency) == "" { + return nil, errors.New("Fetch policy agency is empty") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("SetBucketFetchPolicy", HTTP_PUT, input.Bucket, string(objectKeyExtensionPolicy), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketFetchPolicy gets the bucket fetch policy. +// +// You can use this API to obtain the fetch policy of a bucket. +func (obsClient ObsClient) GetBucketFetchPolicy(input *GetBucketFetchPolicyInput, extensions ...extensionOptions) (output *GetBucketFetchPolicyOutput, err error) { + if input == nil { + return nil, errors.New("GetBucketFetchPolicyInput is nil") + } + output = &GetBucketFetchPolicyOutput{} + err = obsClient.doActionWithBucketAndKeyV2("GetBucketFetchPolicy", HTTP_GET, input.Bucket, string(objectKeyExtensionPolicy), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// DeleteBucketFetchPolicy deletes the bucket fetch policy. +// +// You can use this API to delete the fetch policy of a bucket. +func (obsClient ObsClient) DeleteBucketFetchPolicy(input *DeleteBucketFetchPolicyInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("DeleteBucketFetchPolicyInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("DeleteBucketFetchPolicy", HTTP_DELETE, input.Bucket, string(objectKeyExtensionPolicy), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// SetBucketFetchJob sets the bucket fetch job. +// +// You can use this API to set a bucket fetch job. +func (obsClient ObsClient) SetBucketFetchJob(input *SetBucketFetchJobInput, extensions ...extensionOptions) (output *SetBucketFetchJobOutput, err error) { + if input == nil { + return nil, errors.New("SetBucketFetchJobInput is nil") + } + if strings.TrimSpace(input.URL) == "" { + return nil, errors.New("URL is empty") + } + output = &SetBucketFetchJobOutput{} + err = obsClient.doActionWithBucketAndKeyV2("SetBucketFetchJob", HTTP_POST, input.Bucket, string(objectKeyAsyncFetchJob), input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetBucketFetchJob gets the bucket fetch job. +// +// You can use this API to obtain the fetch job of a bucket. +func (obsClient ObsClient) GetBucketFetchJob(input *GetBucketFetchJobInput, extensions ...extensionOptions) (output *GetBucketFetchJobOutput, err error) { + if input == nil { + return nil, errors.New("GetBucketFetchJobInput is nil") + } + if strings.TrimSpace(input.JobID) == "" { + return nil, errors.New("JobID is empty") + } + output = &GetBucketFetchJobOutput{} + err = obsClient.doActionWithBucketAndKeyV2("GetBucketFetchJob", HTTP_GET, input.Bucket, string(objectKeyAsyncFetchJob)+"/"+input.JobID, input, output, extensions) + if err != nil { + output = nil + } + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_object.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_object.go new file mode 100644 index 0000000000..d68aec9a03 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_object.go @@ -0,0 +1,408 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" + "io" + "os" + "strings" +) + +// ListObjects lists objects in a bucket. +// +// You can use this API to list objects in a bucket. By default, a maximum of 1000 objects are listed. +func (obsClient ObsClient) ListObjects(input *ListObjectsInput, extensions ...extensionOptions) (output *ListObjectsOutput, err error) { + if input == nil { + return nil, errors.New("ListObjectsInput is nil") + } + output = &ListObjectsOutput{} + err = obsClient.doActionWithBucket("ListObjects", HTTP_GET, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + if output.EncodingType == "url" { + err = decodeListObjectsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListObjectsOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// ListVersions lists versioning objects in a bucket. +// +// You can use this API to list versioning objects in a bucket. By default, a maximum of 1000 versioning objects are listed. +func (obsClient ObsClient) ListVersions(input *ListVersionsInput, extensions ...extensionOptions) (output *ListVersionsOutput, err error) { + if input == nil { + return nil, errors.New("ListVersionsInput is nil") + } + output = &ListVersionsOutput{} + err = obsClient.doActionWithBucket("ListVersions", HTTP_GET, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + if output.EncodingType == "url" { + err = decodeListVersionsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListVersionsOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// HeadObject checks whether an object exists. +// +// You can use this API to check whether an object exists. +func (obsClient ObsClient) HeadObject(input *HeadObjectInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("HeadObjectInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("HeadObject", HTTP_HEAD, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// SetObjectMetadata sets object metadata. +func (obsClient ObsClient) SetObjectMetadata(input *SetObjectMetadataInput, extensions ...extensionOptions) (output *SetObjectMetadataOutput, err error) { + output = &SetObjectMetadataOutput{} + err = obsClient.doActionWithBucketAndKey("SetObjectMetadata", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseSetObjectMetadataOutput(output) + } + return +} + +// DeleteObject deletes an object. +// +// You can use this API to delete an object from a specified bucket. +func (obsClient ObsClient) DeleteObject(input *DeleteObjectInput, extensions ...extensionOptions) (output *DeleteObjectOutput, err error) { + if input == nil { + return nil, errors.New("DeleteObjectInput is nil") + } + output = &DeleteObjectOutput{} + err = obsClient.doActionWithBucketAndKey("DeleteObject", HTTP_DELETE, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseDeleteObjectOutput(output) + } + return +} + +// DeleteObjects deletes objects in a batch. +// +// You can use this API to batch delete objects from a specified bucket. +func (obsClient ObsClient) DeleteObjects(input *DeleteObjectsInput, extensions ...extensionOptions) (output *DeleteObjectsOutput, err error) { + if input == nil { + return nil, errors.New("DeleteObjectsInput is nil") + } + output = &DeleteObjectsOutput{} + err = obsClient.doActionWithBucket("DeleteObjects", HTTP_POST, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else if output.EncodingType == "url" { + err = decodeDeleteObjectsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get DeleteObjectsOutput with error: %v.", err) + output = nil + } + } + return +} + +// SetObjectAcl sets ACL for an object. +// +// You can use this API to set the ACL for an object in a specified bucket. +func (obsClient ObsClient) SetObjectAcl(input *SetObjectAclInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetObjectAclInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("SetObjectAcl", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetObjectAcl gets the ACL of an object. +// +// You can use this API to obtain the ACL of an object in a specified bucket. +func (obsClient ObsClient) GetObjectAcl(input *GetObjectAclInput, extensions ...extensionOptions) (output *GetObjectAclOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectAclInput is nil") + } + output = &GetObjectAclOutput{} + err = obsClient.doActionWithBucketAndKey("GetObjectAcl", HTTP_GET, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + if versionID, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionID[0] + } + } + return +} + +// RestoreObject restores an object. +func (obsClient ObsClient) RestoreObject(input *RestoreObjectInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("RestoreObjectInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("RestoreObject", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// GetObjectMetadata gets object metadata. +// +// You can use this API to send a HEAD request to the object of a specified bucket to obtain its metadata. +func (obsClient ObsClient) GetObjectMetadata(input *GetObjectMetadataInput, extensions ...extensionOptions) (output *GetObjectMetadataOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectMetadataInput is nil") + } + output = &GetObjectMetadataOutput{} + err = obsClient.doActionWithBucketAndKey("GetObjectMetadata", HTTP_HEAD, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseGetObjectMetadataOutput(output) + } + return +} + +// GetObject downloads object. +// +// You can use this API to download an object in a specified bucket. +func (obsClient ObsClient) GetObject(input *GetObjectInput, extensions ...extensionOptions) (output *GetObjectOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectInput is nil") + } + output = &GetObjectOutput{} + err = obsClient.doActionWithBucketAndKey("GetObject", HTTP_GET, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseGetObjectOutput(output) + } + return +} + +// PutObject uploads an object to the specified bucket. +func (obsClient ObsClient) PutObject(input *PutObjectInput, extensions ...extensionOptions) (output *PutObjectOutput, err error) { + if input == nil { + return nil, errors.New("PutObjectInput is nil") + } + + if input.ContentType == "" && input.Key != "" { + if contentType, ok := mimeTypes[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + input.ContentType = contentType + } + } + output = &PutObjectOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if input.ContentLength > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.ContentLength} + } + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +func (obsClient ObsClient) getContentType(input *PutObjectInput, sourceFile string) (contentType string) { + if contentType, ok := mimeTypes[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + return contentType + } + if contentType, ok := mimeTypes[strings.ToLower(sourceFile[strings.LastIndex(sourceFile, ".")+1:])]; ok { + return contentType + } + return +} + +func (obsClient ObsClient) isGetContentType(input *PutObjectInput) bool { + if input.ContentType == "" && input.Key != "" { + return true + } + return false +} + +// PutFile uploads a file to the specified bucket. +func (obsClient ObsClient) PutFile(input *PutFileInput, extensions ...extensionOptions) (output *PutObjectOutput, err error) { + if input == nil { + return nil, errors.New("PutFileInput is nil") + } + + var body io.Reader + sourceFile := strings.TrimSpace(input.SourceFile) + if sourceFile != "" { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err + return nil, err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() + + stat, _err := fd.Stat() + if _err != nil { + err = _err + return nil, err + } + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + if input.ContentLength > 0 { + if input.ContentLength > stat.Size() { + input.ContentLength = stat.Size() + } + fileReaderWrapper.totalCount = input.ContentLength + } else { + fileReaderWrapper.totalCount = stat.Size() + } + body = fileReaderWrapper + } + + _input := &PutObjectInput{} + _input.PutObjectBasicInput = input.PutObjectBasicInput + _input.Body = body + + if obsClient.isGetContentType(_input) { + _input.ContentType = obsClient.getContentType(_input, sourceFile) + } + + output = &PutObjectOutput{} + err = obsClient.doActionWithBucketAndKey("PutFile", HTTP_PUT, _input.Bucket, _input.Key, _input, output, extensions) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +// CopyObject creates a copy for an existing object. +// +// You can use this API to create a copy for an object in a specified bucket. +func (obsClient ObsClient) CopyObject(input *CopyObjectInput, extensions ...extensionOptions) (output *CopyObjectOutput, err error) { + if input == nil { + return nil, errors.New("CopyObjectInput is nil") + } + + if strings.TrimSpace(input.CopySourceBucket) == "" { + return nil, errors.New("Source bucket is empty") + } + if strings.TrimSpace(input.CopySourceKey) == "" { + return nil, errors.New("Source key is empty") + } + + output = &CopyObjectOutput{} + err = obsClient.doActionWithBucketAndKey("CopyObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseCopyObjectOutput(output) + } + return +} + +func (obsClient ObsClient) AppendObject(input *AppendObjectInput, extensions ...extensionOptions) (output *AppendObjectOutput, err error) { + if input == nil { + return nil, errors.New("AppendObjectInput is nil") + } + + if input.ContentType == "" && input.Key != "" { + if contentType, ok := mimeTypes[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + input.ContentType = contentType + } + } + output = &AppendObjectOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if input.ContentLength > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.ContentLength} + } + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("AppendObject", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("AppendObject", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + } + if err != nil { + output = nil + } else { + if err = ParseAppendObjectOutput(output); err != nil { + output = nil + } + } + return +} + +func (obsClient ObsClient) ModifyObject(input *ModifyObjectInput, extensions ...extensionOptions) (output *ModifyObjectOutput, err error) { + if input == nil { + return nil, errors.New("ModifyObjectInput is nil") + } + + output = &ModifyObjectOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if input.ContentLength > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.ContentLength} + } + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("ModifyObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("ModifyObject", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } + if err != nil { + output = nil + } else { + ParseModifyObjectOutput(output) + } + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_other.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_other.go new file mode 100644 index 0000000000..e6e3584ef0 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_other.go @@ -0,0 +1,49 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "strings" +) + +// Refresh refreshes ak, sk and securityToken for obsClient. +func (obsClient ObsClient) Refresh(ak, sk, securityToken string) { + for _, sp := range obsClient.conf.securityProviders { + if bsp, ok := sp.(*BasicSecurityProvider); ok { + bsp.refresh(strings.TrimSpace(ak), strings.TrimSpace(sk), strings.TrimSpace(securityToken)) + break + } + } +} + +func (obsClient ObsClient) getSecurity() securityHolder { + if obsClient.conf.securityProviders != nil { + for _, sp := range obsClient.conf.securityProviders { + if sp == nil { + continue + } + sh := sp.getSecurity() + if sh.ak != "" && sh.sk != "" { + return sh + } + } + } + return emptySecurityHolder +} + +// Close closes ObsClient. +func (obsClient ObsClient) Close() { + obsClient.httpClient = nil + obsClient.conf.transport.CloseIdleConnections() + obsClient.conf = nil +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_part.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_part.go new file mode 100644 index 0000000000..f83a0d1b77 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_part.go @@ -0,0 +1,250 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" + "io" + "os" + "sort" + "strings" +) + +// ListMultipartUploads lists the multipart uploads. +// +// You can use this API to list the multipart uploads that are initialized but not combined or aborted in a specified bucket. +func (obsClient ObsClient) ListMultipartUploads(input *ListMultipartUploadsInput, extensions ...extensionOptions) (output *ListMultipartUploadsOutput, err error) { + if input == nil { + return nil, errors.New("ListMultipartUploadsInput is nil") + } + output = &ListMultipartUploadsOutput{} + err = obsClient.doActionWithBucket("ListMultipartUploads", HTTP_GET, input.Bucket, input, output, extensions) + if err != nil { + output = nil + } else if output.EncodingType == "url" { + err = decodeListMultipartUploadsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListMultipartUploadsOutput with error: %v.", err) + output = nil + } + } + return +} + +// AbortMultipartUpload aborts a multipart upload in a specified bucket by using the multipart upload ID. +func (obsClient ObsClient) AbortMultipartUpload(input *AbortMultipartUploadInput, extensions ...extensionOptions) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("AbortMultipartUploadInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("AbortMultipartUpload", HTTP_DELETE, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } + return +} + +// InitiateMultipartUpload initializes a multipart upload. +func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploadInput, extensions ...extensionOptions) (output *InitiateMultipartUploadOutput, err error) { + if input == nil { + return nil, errors.New("InitiateMultipartUploadInput is nil") + } + + if input.ContentType == "" && input.Key != "" { + if contentType, ok := mimeTypes[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + input.ContentType = contentType + } + } + + output = &InitiateMultipartUploadOutput{} + err = obsClient.doActionWithBucketAndKey("InitiateMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseInitiateMultipartUploadOutput(output) + if output.EncodingType == "url" { + err = decodeInitiateMultipartUploadOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get InitiateMultipartUploadOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// UploadPart uploads a part to a specified bucket by using a specified multipart upload ID. +// +// After a multipart upload is initialized, you can use this API to upload a part to a specified bucket +// by using the multipart upload ID. Except for the last uploaded part whose size ranges from 0 to 5 GB, +// sizes of the other parts range from 100 KB to 5 GB. The upload part ID ranges from 1 to 10000. +func (obsClient ObsClient) UploadPart(_input *UploadPartInput, extensions ...extensionOptions) (output *UploadPartOutput, err error) { + if _input == nil { + return nil, errors.New("UploadPartInput is nil") + } + + if _input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + + input := &UploadPartInput{} + input.Bucket = _input.Bucket + input.Key = _input.Key + input.PartNumber = _input.PartNumber + input.UploadId = _input.UploadId + input.ContentMD5 = _input.ContentMD5 + input.SourceFile = _input.SourceFile + input.Offset = _input.Offset + input.PartSize = _input.PartSize + input.SseHeader = _input.SseHeader + input.Body = _input.Body + + output = &UploadPartOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if _, ok := input.Body.(*readerWrapper); !ok && input.PartSize > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.PartSize} + } + } else if sourceFile := strings.TrimSpace(input.SourceFile); sourceFile != "" { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err + return nil, err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() + + stat, _err := fd.Stat() + if _err != nil { + err = _err + return nil, err + } + fileSize := stat.Size() + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + + if input.Offset < 0 || input.Offset > fileSize { + input.Offset = 0 + } + + if input.PartSize <= 0 || input.PartSize > (fileSize-input.Offset) { + input.PartSize = fileSize - input.Offset + } + fileReaderWrapper.totalCount = input.PartSize + if _, err = fd.Seek(input.Offset, io.SeekStart); err != nil { + return nil, err + } + input.Body = fileReaderWrapper + repeatable = true + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + } + if err != nil { + output = nil + } else { + ParseUploadPartOutput(output) + output.PartNumber = input.PartNumber + } + return +} + +// CompleteMultipartUpload combines the uploaded parts in a specified bucket by using the multipart upload ID. +func (obsClient ObsClient) CompleteMultipartUpload(input *CompleteMultipartUploadInput, extensions ...extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + if input == nil { + return nil, errors.New("CompleteMultipartUploadInput is nil") + } + + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + + var parts partSlice = input.Parts + sort.Sort(parts) + + output = &CompleteMultipartUploadOutput{} + err = obsClient.doActionWithBucketAndKey("CompleteMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseCompleteMultipartUploadOutput(output) + if output.EncodingType == "url" { + err = decodeCompleteMultipartUploadOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get CompleteMultipartUploadOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// ListParts lists the uploaded parts in a bucket by using the multipart upload ID. +func (obsClient ObsClient) ListParts(input *ListPartsInput, extensions ...extensionOptions) (output *ListPartsOutput, err error) { + if input == nil { + return nil, errors.New("ListPartsInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + output = &ListPartsOutput{} + err = obsClient.doActionWithBucketAndKey("ListParts", HTTP_GET, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else if output.EncodingType == "url" { + err = decodeListPartsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListPartsOutput with error: %v.", err) + output = nil + } + } + return +} + +// CopyPart copy a part to a specified bucket by using a specified multipart upload ID. +// +// After a multipart upload is initialized, you can use this API to copy a part to a specified bucket by using the multipart upload ID. +func (obsClient ObsClient) CopyPart(input *CopyPartInput, extensions ...extensionOptions) (output *CopyPartOutput, err error) { + if input == nil { + return nil, errors.New("CopyPartInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + if strings.TrimSpace(input.CopySourceBucket) == "" { + return nil, errors.New("Source bucket is empty") + } + if strings.TrimSpace(input.CopySourceKey) == "" { + return nil, errors.New("Source key is empty") + } + + output = &CopyPartOutput{} + err = obsClient.doActionWithBucketAndKey("CopyPart", HTTP_PUT, input.Bucket, input.Key, input, output, extensions) + if err != nil { + output = nil + } else { + ParseCopyPartOutput(output) + output.PartNumber = input.PartNumber + } + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_resume.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_resume.go new file mode 100644 index 0000000000..aef1b4ca79 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/client_resume.go @@ -0,0 +1,59 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +// UploadFile resume uploads. +// +// This API is an encapsulated and enhanced version of multipart upload, and aims to eliminate large file +// upload failures caused by poor network conditions and program breakdowns. +func (obsClient ObsClient) UploadFile(input *UploadFileInput, extensions ...extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + if input.EnableCheckpoint && input.CheckpointFile == "" { + input.CheckpointFile = input.UploadFile + ".uploadfile_record" + } + + if input.TaskNum <= 0 { + input.TaskNum = 1 + } + if input.PartSize < MIN_PART_SIZE { + input.PartSize = MIN_PART_SIZE + } else if input.PartSize > MAX_PART_SIZE { + input.PartSize = MAX_PART_SIZE + } + + output, err = obsClient.resumeUpload(input, extensions) + return +} + +// DownloadFile resume downloads. +// +// This API is an encapsulated and enhanced version of partial download, and aims to eliminate large file +// download failures caused by poor network conditions and program breakdowns. +func (obsClient ObsClient) DownloadFile(input *DownloadFileInput, extensions ...extensionOptions) (output *GetObjectMetadataOutput, err error) { + if input.DownloadFile == "" { + input.DownloadFile = input.Key + } + + if input.EnableCheckpoint && input.CheckpointFile == "" { + input.CheckpointFile = input.DownloadFile + ".downloadfile_record" + } + + if input.TaskNum <= 0 { + input.TaskNum = 1 + } + if input.PartSize <= 0 { + input.PartSize = DEFAULT_PART_SIZE + } + + output, err = obsClient.resumeDownload(input, extensions) + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/conf.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/conf.go new file mode 100644 index 0000000000..1448d03ef3 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/conf.go @@ -0,0 +1,497 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +type urlHolder struct { + scheme string + host string + port int +} + +type config struct { + securityProviders []securityProvider + urlHolder *urlHolder + pathStyle bool + cname bool + sslVerify bool + endpoint string + signature SignatureType + region string + connectTimeout int + socketTimeout int + headerTimeout int + idleConnTimeout int + finalTimeout int + maxRetryCount int + proxyURL string + maxConnsPerHost int + pemCerts []byte + transport *http.Transport + ctx context.Context + maxRedirectCount int + userAgent string + enableCompression bool +} + +func (conf config) String() string { + return fmt.Sprintf("[endpoint:%s, signature:%s, pathStyle:%v, region:%s"+ + "\nconnectTimeout:%d, socketTimeout:%dheaderTimeout:%d, idleConnTimeout:%d"+ + "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, maxRedirectCount:%d]", + conf.endpoint, conf.signature, conf.pathStyle, conf.region, + conf.connectTimeout, conf.socketTimeout, conf.headerTimeout, conf.idleConnTimeout, + conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.maxRedirectCount, + ) +} + +type configurer func(conf *config) + +func WithSecurityProviders(sps ...securityProvider) configurer { + return func(conf *config) { + for _, sp := range sps { + if sp != nil { + conf.securityProviders = append(conf.securityProviders, sp) + } + } + } +} + +// WithSslVerify is a wrapper for WithSslVerifyAndPemCerts. +func WithSslVerify(sslVerify bool) configurer { + return WithSslVerifyAndPemCerts(sslVerify, nil) +} + +// WithSslVerifyAndPemCerts is a configurer for ObsClient to set conf.sslVerify and conf.pemCerts. +func WithSslVerifyAndPemCerts(sslVerify bool, pemCerts []byte) configurer { + return func(conf *config) { + conf.sslVerify = sslVerify + conf.pemCerts = pemCerts + } +} + +// WithHeaderTimeout is a configurer for ObsClient to set the timeout period of obtaining the response headers. +func WithHeaderTimeout(headerTimeout int) configurer { + return func(conf *config) { + conf.headerTimeout = headerTimeout + } +} + +// WithProxyUrl is a configurer for ObsClient to set HTTP proxy. +func WithProxyUrl(proxyURL string) configurer { + return func(conf *config) { + conf.proxyURL = proxyURL + } +} + +// WithMaxConnections is a configurer for ObsClient to set the maximum number of idle HTTP connections. +func WithMaxConnections(maxConnsPerHost int) configurer { + return func(conf *config) { + conf.maxConnsPerHost = maxConnsPerHost + } +} + +// WithPathStyle is a configurer for ObsClient. +func WithPathStyle(pathStyle bool) configurer { + return func(conf *config) { + conf.pathStyle = pathStyle + } +} + +// WithSignature is a configurer for ObsClient. +func WithSignature(signature SignatureType) configurer { + return func(conf *config) { + conf.signature = signature + } +} + +// WithRegion is a configurer for ObsClient. +func WithRegion(region string) configurer { + return func(conf *config) { + conf.region = region + } +} + +// WithConnectTimeout is a configurer for ObsClient to set timeout period for establishing +// an http/https connection, in seconds. +func WithConnectTimeout(connectTimeout int) configurer { + return func(conf *config) { + conf.connectTimeout = connectTimeout + } +} + +// WithSocketTimeout is a configurer for ObsClient to set the timeout duration for transmitting data at +// the socket layer, in seconds. +func WithSocketTimeout(socketTimeout int) configurer { + return func(conf *config) { + conf.socketTimeout = socketTimeout + } +} + +// WithIdleConnTimeout is a configurer for ObsClient to set the timeout period of an idle HTTP connection +// in the connection pool, in seconds. +func WithIdleConnTimeout(idleConnTimeout int) configurer { + return func(conf *config) { + conf.idleConnTimeout = idleConnTimeout + } +} + +// WithMaxRetryCount is a configurer for ObsClient to set the maximum number of retries when an HTTP/HTTPS connection is abnormal. +func WithMaxRetryCount(maxRetryCount int) configurer { + return func(conf *config) { + conf.maxRetryCount = maxRetryCount + } +} + +// WithSecurityToken is a configurer for ObsClient to set the security token in the temporary access keys. +func WithSecurityToken(securityToken string) configurer { + return func(conf *config) { + for _, sp := range conf.securityProviders { + if bsp, ok := sp.(*BasicSecurityProvider); ok { + sh := bsp.getSecurity() + bsp.refresh(sh.ak, sh.sk, securityToken) + break + } + } + } +} + +// WithHttpTransport is a configurer for ObsClient to set the customized http Transport. +func WithHttpTransport(transport *http.Transport) configurer { + return func(conf *config) { + conf.transport = transport + } +} + +// WithRequestContext is a configurer for ObsClient to set the context for each HTTP request. +func WithRequestContext(ctx context.Context) configurer { + return func(conf *config) { + conf.ctx = ctx + } +} + +// WithCustomDomainName is a configurer for ObsClient. +func WithCustomDomainName(cname bool) configurer { + return func(conf *config) { + conf.cname = cname + } +} + +// WithMaxRedirectCount is a configurer for ObsClient to set the maximum number of times that the request is redirected. +func WithMaxRedirectCount(maxRedirectCount int) configurer { + return func(conf *config) { + conf.maxRedirectCount = maxRedirectCount + } +} + +// WithUserAgent is a configurer for ObsClient to set the User-Agent. +func WithUserAgent(userAgent string) configurer { + return func(conf *config) { + conf.userAgent = userAgent + } +} + +// WithEnableCompression is a configurer for ObsClient to set the Transport.DisableCompression. +func WithEnableCompression(enableCompression bool) configurer { + return func(conf *config) { + conf.enableCompression = enableCompression + } +} + +func (conf *config) prepareConfig() { + if conf.connectTimeout <= 0 { + conf.connectTimeout = DEFAULT_CONNECT_TIMEOUT + } + + if conf.socketTimeout <= 0 { + conf.socketTimeout = DEFAULT_SOCKET_TIMEOUT + } + + conf.finalTimeout = conf.socketTimeout * 10 + + if conf.headerTimeout <= 0 { + conf.headerTimeout = DEFAULT_HEADER_TIMEOUT + } + + if conf.idleConnTimeout < 0 { + conf.idleConnTimeout = DEFAULT_IDLE_CONN_TIMEOUT + } + + if conf.maxRetryCount < 0 { + conf.maxRetryCount = DEFAULT_MAX_RETRY_COUNT + } + + if conf.maxConnsPerHost <= 0 { + conf.maxConnsPerHost = DEFAULT_MAX_CONN_PER_HOST + } + + if conf.maxRedirectCount < 0 { + conf.maxRedirectCount = DEFAULT_MAX_REDIRECT_COUNT + } + + if conf.pathStyle && conf.signature == SignatureObs { + conf.signature = SignatureV2 + } +} + +func (conf *config) initConfigWithDefault() error { + conf.endpoint = strings.TrimSpace(conf.endpoint) + if conf.endpoint == "" { + return errors.New("endpoint is not set") + } + + if index := strings.Index(conf.endpoint, "?"); index > 0 { + conf.endpoint = conf.endpoint[:index] + } + + for strings.LastIndex(conf.endpoint, "/") == len(conf.endpoint)-1 { + conf.endpoint = conf.endpoint[:len(conf.endpoint)-1] + } + + if conf.signature == "" { + conf.signature = DEFAULT_SIGNATURE + } + + urlHolder := &urlHolder{} + var address string + if strings.HasPrefix(conf.endpoint, "https://") { + urlHolder.scheme = "https" + address = conf.endpoint[len("https://"):] + } else if strings.HasPrefix(conf.endpoint, "http://") { + urlHolder.scheme = "http" + address = conf.endpoint[len("http://"):] + } else { + urlHolder.scheme = "https" + address = conf.endpoint + } + + addr := strings.Split(address, ":") + if len(addr) == 2 { + if port, err := strconv.Atoi(addr[1]); err == nil { + urlHolder.port = port + } + } + urlHolder.host = addr[0] + if urlHolder.port == 0 { + if urlHolder.scheme == "https" { + urlHolder.port = 443 + } else { + urlHolder.port = 80 + } + } + + if IsIP(urlHolder.host) { + conf.pathStyle = true + } + + conf.urlHolder = urlHolder + + conf.region = strings.TrimSpace(conf.region) + if conf.region == "" { + conf.region = DEFAULT_REGION + } + + conf.prepareConfig() + conf.proxyURL = strings.TrimSpace(conf.proxyURL) + return nil +} + +func (conf *config) getTransport() error { + if conf.transport == nil { + conf.transport = &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(network, addr, time.Second*time.Duration(conf.connectTimeout)) + if err != nil { + return nil, err + } + return getConnDelegate(conn, conf.socketTimeout, conf.finalTimeout), nil + }, + MaxIdleConns: conf.maxConnsPerHost, + MaxIdleConnsPerHost: conf.maxConnsPerHost, + ResponseHeaderTimeout: time.Second * time.Duration(conf.headerTimeout), + IdleConnTimeout: time.Second * time.Duration(conf.idleConnTimeout), + } + + if conf.proxyURL != "" { + proxyURL, err := url.Parse(conf.proxyURL) + if err != nil { + return err + } + conf.transport.Proxy = http.ProxyURL(proxyURL) + } + + tlsConfig := &tls.Config{InsecureSkipVerify: !conf.sslVerify} + if conf.sslVerify && conf.pemCerts != nil { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(conf.pemCerts) + tlsConfig.RootCAs = pool + } + + conf.transport.TLSClientConfig = tlsConfig + conf.transport.DisableCompression = !conf.enableCompression + } + + return nil +} + +func checkRedirectFunc(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse +} + +// DummyQueryEscape return the input string. +func DummyQueryEscape(s string) string { + return s +} + +func (conf *config) prepareBaseURL(bucketName string) (requestURL string, canonicalizedURL string) { + urlHolder := conf.urlHolder + if conf.cname { + requestURL = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) + if conf.signature == "v4" { + canonicalizedURL = "/" + } else { + canonicalizedURL = "/" + urlHolder.host + "/" + } + } else { + if bucketName == "" { + requestURL = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) + canonicalizedURL = "/" + } else { + if conf.pathStyle { + requestURL = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName) + canonicalizedURL = "/" + bucketName + } else { + requestURL = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port) + if conf.signature == "v2" || conf.signature == "OBS" { + canonicalizedURL = "/" + bucketName + "/" + } else { + canonicalizedURL = "/" + } + } + } + } + return +} + +func (conf *config) prepareObjectKey(escape bool, objectKey string, escapeFunc func(s string) string) (encodeObjectKey string) { + if escape { + tempKey := []rune(objectKey) + result := make([]string, 0, len(tempKey)) + for _, value := range tempKey { + if string(value) == "/" { + result = append(result, string(value)) + } else { + if string(value) == " " { + result = append(result, url.PathEscape(string(value))) + } else { + result = append(result, url.QueryEscape(string(value))) + } + } + } + encodeObjectKey = strings.Join(result, "") + } else { + encodeObjectKey = escapeFunc(objectKey) + } + return +} + +func (conf *config) prepareEscapeFunc(escape bool) (escapeFunc func(s string) string) { + if escape { + return url.QueryEscape + } + return DummyQueryEscape +} + +func (conf *config) formatUrls(bucketName, objectKey string, params map[string]string, escape bool) (requestURL string, canonicalizedURL string) { + + requestURL, canonicalizedURL = conf.prepareBaseURL(bucketName) + var escapeFunc func(s string) string + escapeFunc = conf.prepareEscapeFunc(escape) + + if objectKey != "" { + var encodeObjectKey string + encodeObjectKey = conf.prepareObjectKey(escape, objectKey, escapeFunc) + requestURL += "/" + encodeObjectKey + if !strings.HasSuffix(canonicalizedURL, "/") { + canonicalizedURL += "/" + } + canonicalizedURL += encodeObjectKey + } + + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, strings.TrimSpace(key)) + } + sort.Strings(keys) + i := 0 + + for index, key := range keys { + if index == 0 { + requestURL += "?" + } else { + requestURL += "&" + } + _key := url.QueryEscape(key) + requestURL += _key + + _value := params[key] + if conf.signature == "v4" { + requestURL += "=" + url.QueryEscape(_value) + } else { + if _value != "" { + requestURL += "=" + url.QueryEscape(_value) + _value = "=" + _value + } else { + _value = "" + } + lowerKey := strings.ToLower(key) + _, ok := allowedResourceParameterNames[lowerKey] + prefixHeader := HEADER_PREFIX + isObs := conf.signature == SignatureObs + if isObs { + prefixHeader = HEADER_PREFIX_OBS + } + ok = ok || strings.HasPrefix(lowerKey, prefixHeader) + if ok { + if i == 0 { + canonicalizedURL += "?" + } else { + canonicalizedURL += "&" + } + canonicalizedURL += getQueryURL(_key, _value) + i++ + } + } + } + return +} + +func getQueryURL(key, value string) string { + queryURL := "" + queryURL += key + queryURL += value + return queryURL +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/const.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/const.go new file mode 100644 index 0000000000..8aabd6e4ac --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/const.go @@ -0,0 +1,969 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +const ( + obsSdkVersion = "3.21.8" + USER_AGENT = "obs-sdk-go/" + obsSdkVersion + HEADER_PREFIX = "x-amz-" + HEADER_PREFIX_META = "x-amz-meta-" + HEADER_PREFIX_OBS = "x-obs-" + HEADER_PREFIX_META_OBS = "x-obs-meta-" + HEADER_DATE_AMZ = "x-amz-date" + HEADER_DATE_OBS = "x-obs-date" + HEADER_STS_TOKEN_AMZ = "x-amz-security-token" + HEADER_STS_TOKEN_OBS = "x-obs-security-token" + HEADER_ACCESSS_KEY_AMZ = "AWSAccessKeyId" + PREFIX_META = "meta-" + + HEADER_CONTENT_SHA256_AMZ = "x-amz-content-sha256" + HEADER_ACL_AMZ = "x-amz-acl" + HEADER_ACL_OBS = "x-obs-acl" + HEADER_ACL = "acl" + HEADER_LOCATION_AMZ = "location" + HEADER_BUCKET_LOCATION_OBS = "bucket-location" + HEADER_COPY_SOURCE = "copy-source" + HEADER_COPY_SOURCE_RANGE = "copy-source-range" + HEADER_RANGE = "Range" + HEADER_STORAGE_CLASS = "x-default-storage-class" + HEADER_STORAGE_CLASS_OBS = "x-obs-storage-class" + HEADER_VERSION_OBS = "version" + HEADER_GRANT_READ_OBS = "grant-read" + HEADER_GRANT_WRITE_OBS = "grant-write" + HEADER_GRANT_READ_ACP_OBS = "grant-read-acp" + HEADER_GRANT_WRITE_ACP_OBS = "grant-write-acp" + HEADER_GRANT_FULL_CONTROL_OBS = "grant-full-control" + HEADER_GRANT_READ_DELIVERED_OBS = "grant-read-delivered" + HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS = "grant-full-control-delivered" + HEADER_REQUEST_ID = "request-id" + HEADER_BUCKET_REGION = "bucket-region" + HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN = "access-control-allow-origin" + HEADER_ACCESS_CONRTOL_ALLOW_HEADERS = "access-control-allow-headers" + HEADER_ACCESS_CONRTOL_MAX_AGE = "access-control-max-age" + HEADER_ACCESS_CONRTOL_ALLOW_METHODS = "access-control-allow-methods" + HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS = "access-control-expose-headers" + HEADER_EPID_HEADERS = "epid" + HEADER_VERSION_ID = "version-id" + HEADER_COPY_SOURCE_VERSION_ID = "copy-source-version-id" + HEADER_DELETE_MARKER = "delete-marker" + HEADER_WEBSITE_REDIRECT_LOCATION = "website-redirect-location" + HEADER_METADATA_DIRECTIVE = "metadata-directive" + HEADER_EXPIRATION = "expiration" + HEADER_EXPIRES_OBS = "x-obs-expires" + HEADER_RESTORE = "restore" + HEADER_OBJECT_TYPE = "object-type" + HEADER_NEXT_APPEND_POSITION = "next-append-position" + HEADER_STORAGE_CLASS2 = "storage-class" + HEADER_CONTENT_LENGTH = "content-length" + HEADER_CONTENT_TYPE = "content-type" + HEADER_CONTENT_LANGUAGE = "content-language" + HEADER_EXPIRES = "expires" + HEADER_CACHE_CONTROL = "cache-control" + HEADER_CONTENT_DISPOSITION = "content-disposition" + HEADER_CONTENT_ENCODING = "content-encoding" + HEADER_AZ_REDUNDANCY = "az-redundancy" + HEADER_BUCKET_TYPE = "bucket-type" + headerOefMarker = "oef-marker" + + HEADER_ETAG = "etag" + HEADER_LASTMODIFIED = "last-modified" + + HEADER_COPY_SOURCE_IF_MATCH = "copy-source-if-match" + HEADER_COPY_SOURCE_IF_NONE_MATCH = "copy-source-if-none-match" + HEADER_COPY_SOURCE_IF_MODIFIED_SINCE = "copy-source-if-modified-since" + HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE = "copy-source-if-unmodified-since" + + HEADER_IF_MATCH = "If-Match" + HEADER_IF_NONE_MATCH = "If-None-Match" + HEADER_IF_MODIFIED_SINCE = "If-Modified-Since" + HEADER_IF_UNMODIFIED_SINCE = "If-Unmodified-Since" + + HEADER_SSEC_ENCRYPTION = "server-side-encryption-customer-algorithm" + HEADER_SSEC_KEY = "server-side-encryption-customer-key" + HEADER_SSEC_KEY_MD5 = "server-side-encryption-customer-key-MD5" + + HEADER_SSEKMS_ENCRYPTION = "server-side-encryption" + HEADER_SSEKMS_KEY = "server-side-encryption-aws-kms-key-id" + HEADER_SSEKMS_ENCRYPT_KEY_OBS = "server-side-encryption-kms-key-id" + + HEADER_SSEC_COPY_SOURCE_ENCRYPTION = "copy-source-server-side-encryption-customer-algorithm" + HEADER_SSEC_COPY_SOURCE_KEY = "copy-source-server-side-encryption-customer-key" + HEADER_SSEC_COPY_SOURCE_KEY_MD5 = "copy-source-server-side-encryption-customer-key-MD5" + + HEADER_SSEKMS_KEY_AMZ = "x-amz-server-side-encryption-aws-kms-key-id" + + HEADER_SSEKMS_KEY_OBS = "x-obs-server-side-encryption-kms-key-id" + + HEADER_SUCCESS_ACTION_REDIRECT = "success_action_redirect" + + headerFSFileInterface = "fs-file-interface" + + HEADER_DATE_CAMEL = "Date" + HEADER_HOST_CAMEL = "Host" + HEADER_HOST = "host" + HEADER_AUTH_CAMEL = "Authorization" + HEADER_MD5_CAMEL = "Content-MD5" + HEADER_LOCATION_CAMEL = "Location" + HEADER_CONTENT_LENGTH_CAMEL = "Content-Length" + HEADER_CONTENT_TYPE_CAML = "Content-Type" + HEADER_USER_AGENT_CAMEL = "User-Agent" + HEADER_ORIGIN_CAMEL = "Origin" + HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL = "Access-Control-Request-Headers" + HEADER_CACHE_CONTROL_CAMEL = "Cache-Control" + HEADER_CONTENT_DISPOSITION_CAMEL = "Content-Disposition" + HEADER_CONTENT_ENCODING_CAMEL = "Content-Encoding" + HEADER_CONTENT_LANGUAGE_CAMEL = "Content-Language" + HEADER_EXPIRES_CAMEL = "Expires" + + PARAM_VERSION_ID = "versionId" + PARAM_RESPONSE_CONTENT_TYPE = "response-content-type" + PARAM_RESPONSE_CONTENT_LANGUAGE = "response-content-language" + PARAM_RESPONSE_EXPIRES = "response-expires" + PARAM_RESPONSE_CACHE_CONTROL = "response-cache-control" + PARAM_RESPONSE_CONTENT_DISPOSITION = "response-content-disposition" + PARAM_RESPONSE_CONTENT_ENCODING = "response-content-encoding" + PARAM_IMAGE_PROCESS = "x-image-process" + + PARAM_ALGORITHM_AMZ_CAMEL = "X-Amz-Algorithm" + PARAM_CREDENTIAL_AMZ_CAMEL = "X-Amz-Credential" + PARAM_DATE_AMZ_CAMEL = "X-Amz-Date" + PARAM_DATE_OBS_CAMEL = "X-Obs-Date" + PARAM_EXPIRES_AMZ_CAMEL = "X-Amz-Expires" + PARAM_SIGNEDHEADERS_AMZ_CAMEL = "X-Amz-SignedHeaders" + PARAM_SIGNATURE_AMZ_CAMEL = "X-Amz-Signature" + + DEFAULT_SIGNATURE = SignatureV2 + DEFAULT_REGION = "region" + DEFAULT_CONNECT_TIMEOUT = 60 + DEFAULT_SOCKET_TIMEOUT = 60 + DEFAULT_HEADER_TIMEOUT = 60 + DEFAULT_IDLE_CONN_TIMEOUT = 30 + DEFAULT_MAX_RETRY_COUNT = 3 + DEFAULT_MAX_REDIRECT_COUNT = 3 + DEFAULT_MAX_CONN_PER_HOST = 1000 + EMPTY_CONTENT_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD" + LONG_DATE_FORMAT = "20060102T150405Z" + SHORT_DATE_FORMAT = "20060102" + ISO8601_DATE_FORMAT = "2006-01-02T15:04:05Z" + ISO8601_MIDNIGHT_DATE_FORMAT = "2006-01-02T00:00:00Z" + RFC1123_FORMAT = "Mon, 02 Jan 2006 15:04:05 GMT" + + V4_SERVICE_NAME = "s3" + V4_SERVICE_SUFFIX = "aws4_request" + + V2_HASH_PREFIX = "AWS" + OBS_HASH_PREFIX = "OBS" + + V4_HASH_PREFIX = "AWS4-HMAC-SHA256" + V4_HASH_PRE = "AWS4" + + DEFAULT_SSE_KMS_ENCRYPTION = "aws:kms" + DEFAULT_SSE_KMS_ENCRYPTION_OBS = "kms" + + DEFAULT_SSE_C_ENCRYPTION = "AES256" + + HTTP_GET = "GET" + HTTP_POST = "POST" + HTTP_PUT = "PUT" + HTTP_DELETE = "DELETE" + HTTP_HEAD = "HEAD" + HTTP_OPTIONS = "OPTIONS" + + REQUEST_PAYER = "request-payer" + TRAFFIC_LIMIT = "traffic-limit" + MULTI_AZ = "3az" + + MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 + MIN_PART_SIZE = 100 * 1024 + DEFAULT_PART_SIZE = 9 * 1024 * 1024 + MAX_PART_NUM = 10000 +) + +// SignatureType defines type of signature +type SignatureType string + +const ( + // SignatureV2 signature type v2 + SignatureV2 SignatureType = "v2" + // SignatureV4 signature type v4 + SignatureV4 SignatureType = "v4" + // SignatureObs signature type OBS + SignatureObs SignatureType = "OBS" +) + +var ( + interestedHeaders = []string{"content-md5", "content-type", "date"} + + allowedRequestHTTPHeaderMetadataNames = map[string]bool{ + "content-type": true, + "content-md5": true, + "content-length": true, + "content-language": true, + "expires": true, + "origin": true, + "cache-control": true, + "content-disposition": true, + "content-encoding": true, + "access-control-request-method": true, + "access-control-request-headers": true, + "x-default-storage-class": true, + "location": true, + "date": true, + "etag": true, + "range": true, + "host": true, + "if-modified-since": true, + "if-unmodified-since": true, + "if-match": true, + "if-none-match": true, + "last-modified": true, + "content-range": true, + } + + allowedResourceParameterNames = map[string]bool{ + "acl": true, + "backtosource": true, + "metadata": true, + "policy": true, + "torrent": true, + "logging": true, + "location": true, + "storageinfo": true, + "quota": true, + "storageclass": true, + "storagepolicy": true, + "requestpayment": true, + "versions": true, + "versioning": true, + "versionid": true, + "uploads": true, + "uploadid": true, + "partnumber": true, + "website": true, + "notification": true, + "lifecycle": true, + "deletebucket": true, + "delete": true, + "cors": true, + "restore": true, + "encryption": true, + "tagging": true, + "append": true, + "modify": true, + "position": true, + "replication": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, + "x-image-process": true, + "x-oss-process": true, + "x-image-save-bucket": true, + "x-image-save-object": true, + "ignore-sign-in-query": true, + } + + mimeTypes = map[string]string{ + "001": "application/x-001", + "301": "application/x-301", + "323": "text/h323", + "7z": "application/x-7z-compressed", + "906": "application/x-906", + "907": "drawing/907", + "IVF": "video/x-ivf", + "a11": "application/x-a11", + "aac": "audio/x-aac", + "acp": "audio/x-mei-aac", + "ai": "application/postscript", + "aif": "audio/aiff", + "aifc": "audio/aiff", + "aiff": "audio/aiff", + "anv": "application/x-anv", + "apk": "application/vnd.android.package-archive", + "asa": "text/asa", + "asf": "video/x-ms-asf", + "asp": "text/asp", + "asx": "video/x-ms-asf", + "atom": "application/atom+xml", + "au": "audio/basic", + "avi": "video/avi", + "awf": "application/vnd.adobe.workflow", + "biz": "text/xml", + "bmp": "application/x-bmp", + "bot": "application/x-bot", + "bz2": "application/x-bzip2", + "c4t": "application/x-c4t", + "c90": "application/x-c90", + "cal": "application/x-cals", + "cat": "application/vnd.ms-pki.seccat", + "cdf": "application/x-netcdf", + "cdr": "application/x-cdr", + "cel": "application/x-cel", + "cer": "application/x-x509-ca-cert", + "cg4": "application/x-g4", + "cgm": "application/x-cgm", + "cit": "application/x-cit", + "class": "java/*", + "cml": "text/xml", + "cmp": "application/x-cmp", + "cmx": "application/x-cmx", + "cot": "application/x-cot", + "crl": "application/pkix-crl", + "crt": "application/x-x509-ca-cert", + "csi": "application/x-csi", + "css": "text/css", + "csv": "text/csv", + "cu": "application/cu-seeme", + "cut": "application/x-cut", + "dbf": "application/x-dbf", + "dbm": "application/x-dbm", + "dbx": "application/x-dbx", + "dcd": "text/xml", + "dcx": "application/x-dcx", + "deb": "application/x-debian-package", + "der": "application/x-x509-ca-cert", + "dgn": "application/x-dgn", + "dib": "application/x-dib", + "dll": "application/x-msdownload", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dot": "application/msword", + "drw": "application/x-drw", + "dtd": "text/xml", + "dvi": "application/x-dvi", + "dwf": "application/x-dwf", + "dwg": "application/x-dwg", + "dxb": "application/x-dxb", + "dxf": "application/x-dxf", + "edn": "application/vnd.adobe.edn", + "emf": "application/x-emf", + "eml": "message/rfc822", + "ent": "text/xml", + "eot": "application/vnd.ms-fontobject", + "epi": "application/x-epi", + "eps": "application/postscript", + "epub": "application/epub+zip", + "etd": "application/x-ebx", + "etx": "text/x-setext", + "exe": "application/x-msdownload", + "fax": "image/fax", + "fdf": "application/vnd.fdf", + "fif": "application/fractals", + "flac": "audio/flac", + "flv": "video/x-flv", + "fo": "text/xml", + "frm": "application/x-frm", + "g4": "application/x-g4", + "gbr": "application/x-gbr", + "gif": "image/gif", + "gl2": "application/x-gl2", + "gp4": "application/x-gp4", + "gz": "application/gzip", + "hgl": "application/x-hgl", + "hmr": "application/x-hmr", + "hpg": "application/x-hpgl", + "hpl": "application/x-hpl", + "hqx": "application/mac-binhex40", + "hrf": "application/x-hrf", + "hta": "application/hta", + "htc": "text/x-component", + "htm": "text/html", + "html": "text/html", + "htt": "text/webviewhtml", + "htx": "text/html", + "icb": "application/x-icb", + "ico": "application/x-ico", + "ics": "text/calendar", + "iff": "application/x-iff", + "ig4": "application/x-g4", + "igs": "application/x-igs", + "iii": "application/x-iphone", + "img": "application/x-img", + "ini": "text/plain", + "ins": "application/x-internet-signup", + "ipa": "application/vnd.iphone", + "iso": "application/x-iso9660-image", + "isp": "application/x-internet-signup", + "jar": "application/java-archive", + "java": "java/*", + "jfif": "image/jpeg", + "jpe": "image/jpeg", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "js": "application/x-javascript", + "json": "application/json", + "jsp": "text/html", + "la1": "audio/x-liquid-file", + "lar": "application/x-laplayer-reg", + "latex": "application/x-latex", + "lavs": "audio/x-liquid-secure", + "lbm": "application/x-lbm", + "lmsff": "audio/x-la-lms", + "log": "text/plain", + "ls": "application/x-javascript", + "ltr": "application/x-ltr", + "m1v": "video/x-mpeg", + "m2v": "video/x-mpeg", + "m3u": "audio/mpegurl", + "m4a": "audio/mp4", + "m4e": "video/mpeg4", + "m4v": "video/mp4", + "mac": "application/x-mac", + "man": "application/x-troff-man", + "math": "text/xml", + "mdb": "application/msaccess", + "mfp": "application/x-shockwave-flash", + "mht": "message/rfc822", + "mhtml": "message/rfc822", + "mi": "application/x-mi", + "mid": "audio/mid", + "midi": "audio/mid", + "mil": "application/x-mil", + "mml": "text/xml", + "mnd": "audio/x-musicnet-download", + "mns": "audio/x-musicnet-stream", + "mocha": "application/x-javascript", + "mov": "video/quicktime", + "movie": "video/x-sgi-movie", + "mp1": "audio/mp1", + "mp2": "audio/mp2", + "mp2v": "video/mpeg", + "mp3": "audio/mp3", + "mp4": "video/mp4", + "mp4a": "audio/mp4", + "mp4v": "video/mp4", + "mpa": "video/x-mpg", + "mpd": "application/vnd.ms-project", + "mpe": "video/mpeg", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpg4": "video/mp4", + "mpga": "audio/rn-mpeg", + "mpp": "application/vnd.ms-project", + "mps": "video/x-mpeg", + "mpt": "application/vnd.ms-project", + "mpv": "video/mpg", + "mpv2": "video/mpeg", + "mpw": "application/vnd.ms-project", + "mpx": "application/vnd.ms-project", + "mtx": "text/xml", + "mxp": "application/x-mmxp", + "net": "image/pnetvue", + "nrf": "application/x-nrf", + "nws": "message/rfc822", + "odc": "text/x-ms-odc", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "ogv": "video/ogg", + "ogx": "application/ogg", + "out": "application/x-out", + "p10": "application/pkcs10", + "p12": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "p7c": "application/pkcs7-mime", + "p7m": "application/pkcs7-mime", + "p7r": "application/x-pkcs7-certreqresp", + "p7s": "application/pkcs7-signature", + "pbm": "image/x-portable-bitmap", + "pc5": "application/x-pc5", + "pci": "application/x-pci", + "pcl": "application/x-pcl", + "pcx": "application/x-pcx", + "pdf": "application/pdf", + "pdx": "application/vnd.adobe.pdx", + "pfx": "application/x-pkcs12", + "pgl": "application/x-pgl", + "pgm": "image/x-portable-graymap", + "pic": "application/x-pic", + "pko": "application/vnd.ms-pki.pko", + "pl": "application/x-perl", + "plg": "text/html", + "pls": "audio/scpls", + "plt": "application/x-plt", + "png": "image/png", + "pnm": "image/x-portable-anymap", + "pot": "application/vnd.ms-powerpoint", + "ppa": "application/vnd.ms-powerpoint", + "ppm": "application/x-ppm", + "pps": "application/vnd.ms-powerpoint", + "ppt": "application/vnd.ms-powerpoint", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "pr": "application/x-pr", + "prf": "application/pics-rules", + "prn": "application/x-prn", + "prt": "application/x-prt", + "ps": "application/postscript", + "ptn": "application/x-ptn", + "pwz": "application/vnd.ms-powerpoint", + "qt": "video/quicktime", + "r3t": "text/vnd.rn-realtext3d", + "ra": "audio/vnd.rn-realaudio", + "ram": "audio/x-pn-realaudio", + "rar": "application/x-rar-compressed", + "ras": "application/x-ras", + "rat": "application/rat-file", + "rdf": "text/xml", + "rec": "application/vnd.rn-recording", + "red": "application/x-red", + "rgb": "application/x-rgb", + "rjs": "application/vnd.rn-realsystem-rjs", + "rjt": "application/vnd.rn-realsystem-rjt", + "rlc": "application/x-rlc", + "rle": "application/x-rle", + "rm": "application/vnd.rn-realmedia", + "rmf": "application/vnd.adobe.rmf", + "rmi": "audio/mid", + "rmj": "application/vnd.rn-realsystem-rmj", + "rmm": "audio/x-pn-realaudio", + "rmp": "application/vnd.rn-rn_music_package", + "rms": "application/vnd.rn-realmedia-secure", + "rmvb": "application/vnd.rn-realmedia-vbr", + "rmx": "application/vnd.rn-realsystem-rmx", + "rnx": "application/vnd.rn-realplayer", + "rp": "image/vnd.rn-realpix", + "rpm": "audio/x-pn-realaudio-plugin", + "rsml": "application/vnd.rn-rsml", + "rss": "application/rss+xml", + "rt": "text/vnd.rn-realtext", + "rtf": "application/x-rtf", + "rv": "video/vnd.rn-realvideo", + "sam": "application/x-sam", + "sat": "application/x-sat", + "sdp": "application/sdp", + "sdw": "application/x-sdw", + "sgm": "text/sgml", + "sgml": "text/sgml", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "sit": "application/x-stuffit", + "slb": "application/x-slb", + "sld": "application/x-sld", + "slk": "drawing/x-slk", + "smi": "application/smil", + "smil": "application/smil", + "smk": "application/x-smk", + "snd": "audio/basic", + "sol": "text/plain", + "sor": "text/plain", + "spc": "application/x-pkcs7-certificates", + "spl": "application/futuresplash", + "spp": "text/xml", + "ssm": "application/streamingmedia", + "sst": "application/vnd.ms-pki.certstore", + "stl": "application/vnd.ms-pki.stl", + "stm": "text/html", + "sty": "application/x-sty", + "svg": "image/svg+xml", + "swf": "application/x-shockwave-flash", + "tar": "application/x-tar", + "tdf": "application/x-tdf", + "tg4": "application/x-tg4", + "tga": "application/x-tga", + "tif": "image/tiff", + "tiff": "image/tiff", + "tld": "text/xml", + "top": "drawing/x-top", + "torrent": "application/x-bittorrent", + "tsd": "text/xml", + "ttf": "application/x-font-ttf", + "txt": "text/plain", + "uin": "application/x-icq", + "uls": "text/iuls", + "vcf": "text/x-vcard", + "vda": "application/x-vda", + "vdx": "application/vnd.visio", + "vml": "text/xml", + "vpg": "application/x-vpeg005", + "vsd": "application/vnd.visio", + "vss": "application/vnd.visio", + "vst": "application/x-vst", + "vsw": "application/vnd.visio", + "vsx": "application/vnd.visio", + "vtx": "application/vnd.visio", + "vxml": "text/xml", + "wav": "audio/wav", + "wax": "audio/x-ms-wax", + "wb1": "application/x-wb1", + "wb2": "application/x-wb2", + "wb3": "application/x-wb3", + "wbmp": "image/vnd.wap.wbmp", + "webm": "video/webm", + "wiz": "application/msword", + "wk3": "application/x-wk3", + "wk4": "application/x-wk4", + "wkq": "application/x-wkq", + "wks": "application/x-wks", + "wm": "video/x-ms-wm", + "wma": "audio/x-ms-wma", + "wmd": "application/x-ms-wmd", + "wmf": "application/x-wmf", + "wml": "text/vnd.wap.wml", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wmz": "application/x-ms-wmz", + "woff": "application/x-font-woff", + "wp6": "application/x-wp6", + "wpd": "application/x-wpd", + "wpg": "application/x-wpg", + "wpl": "application/vnd.ms-wpl", + "wq1": "application/x-wq1", + "wr1": "application/x-wr1", + "wri": "application/x-wri", + "wrk": "application/x-wrk", + "ws": "application/x-ws", + "ws2": "application/x-ws", + "wsc": "text/scriptlet", + "wsdl": "text/xml", + "wvx": "video/x-ms-wvx", + "x_b": "application/x-x_b", + "x_t": "application/x-x_t", + "xap": "application/x-silverlight-app", + "xbm": "image/x-xbitmap", + "xdp": "application/vnd.adobe.xdp", + "xdr": "text/xml", + "xfd": "application/vnd.adobe.xfd", + "xfdf": "application/vnd.adobe.xfdf", + "xhtml": "text/html", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlw": "application/x-xlw", + "xml": "text/xml", + "xpl": "audio/scpls", + "xpm": "image/x-xpixmap", + "xq": "text/xml", + "xql": "text/xml", + "xquery": "text/xml", + "xsd": "text/xml", + "xsl": "text/xml", + "xslt": "text/xml", + "xwd": "application/x-xwd", + "yaml": "text/yaml", + "yml": "text/yaml", + "zip": "application/zip", + } +) + +// HttpMethodType defines http method type +type HttpMethodType string + +const ( + HttpMethodGet HttpMethodType = HTTP_GET + HttpMethodPut HttpMethodType = HTTP_PUT + HttpMethodPost HttpMethodType = HTTP_POST + HttpMethodDelete HttpMethodType = HTTP_DELETE + HttpMethodHead HttpMethodType = HTTP_HEAD + HttpMethodOptions HttpMethodType = HTTP_OPTIONS +) + +// SubResourceType defines the subResource value +type SubResourceType string + +const ( + // SubResourceStoragePolicy subResource value: storagePolicy + SubResourceStoragePolicy SubResourceType = "storagePolicy" + + // SubResourceStorageClass subResource value: storageClass + SubResourceStorageClass SubResourceType = "storageClass" + + // SubResourceQuota subResource value: quota + SubResourceQuota SubResourceType = "quota" + + // SubResourceStorageInfo subResource value: storageinfo + SubResourceStorageInfo SubResourceType = "storageinfo" + + // SubResourceLocation subResource value: location + SubResourceLocation SubResourceType = "location" + + // SubResourceAcl subResource value: acl + SubResourceAcl SubResourceType = "acl" + + // SubResourcePolicy subResource value: policy + SubResourcePolicy SubResourceType = "policy" + + // SubResourceCors subResource value: cors + SubResourceCors SubResourceType = "cors" + + // SubResourceVersioning subResource value: versioning + SubResourceVersioning SubResourceType = "versioning" + + // SubResourceWebsite subResource value: website + SubResourceWebsite SubResourceType = "website" + + // SubResourceLogging subResource value: logging + SubResourceLogging SubResourceType = "logging" + + // SubResourceLifecycle subResource value: lifecycle + SubResourceLifecycle SubResourceType = "lifecycle" + + // SubResourceNotification subResource value: notification + SubResourceNotification SubResourceType = "notification" + + // SubResourceEncryption subResource value: encryption + SubResourceEncryption SubResourceType = "encryption" + + // SubResourceTagging subResource value: tagging + SubResourceTagging SubResourceType = "tagging" + + // SubResourceDelete subResource value: delete + SubResourceDelete SubResourceType = "delete" + + // SubResourceVersions subResource value: versions + SubResourceVersions SubResourceType = "versions" + + // SubResourceUploads subResource value: uploads + SubResourceUploads SubResourceType = "uploads" + + // SubResourceRestore subResource value: restore + SubResourceRestore SubResourceType = "restore" + + // SubResourceMetadata subResource value: metadata + SubResourceMetadata SubResourceType = "metadata" + + // SubResourceRequestPayment subResource value: requestPayment + SubResourceRequestPayment SubResourceType = "requestPayment" + + // SubResourceAppend subResource value: append + SubResourceAppend SubResourceType = "append" + + // SubResourceModify subResource value: modify + SubResourceModify SubResourceType = "modify" +) + +// objectKeyType defines the objectKey value +type objectKeyType string + +const ( + // objectKeyExtensionPolicy objectKey value: v1/extension_policy + objectKeyExtensionPolicy objectKeyType = "v1/extension_policy" + + // objectKeyAsyncFetchJob objectKey value: v1/async-fetch/jobs + objectKeyAsyncFetchJob objectKeyType = "v1/async-fetch/jobs" +) + +// AclType defines bucket/object acl type +type AclType string + +const ( + AclPrivate AclType = "private" + AclPublicRead AclType = "public-read" + AclPublicReadWrite AclType = "public-read-write" + AclAuthenticatedRead AclType = "authenticated-read" + AclBucketOwnerRead AclType = "bucket-owner-read" + AclBucketOwnerFullControl AclType = "bucket-owner-full-control" + AclLogDeliveryWrite AclType = "log-delivery-write" + AclPublicReadDelivery AclType = "public-read-delivered" + AclPublicReadWriteDelivery AclType = "public-read-write-delivered" +) + +// StorageClassType defines bucket storage class +type StorageClassType string + +const ( + //StorageClassStandard storage class: STANDARD + StorageClassStandard StorageClassType = "STANDARD" + + //StorageClassWarm storage class: WARM + StorageClassWarm StorageClassType = "WARM" + + //StorageClassCold storage class: COLD + StorageClassCold StorageClassType = "COLD" + + storageClassStandardIA StorageClassType = "STANDARD_IA" + storageClassGlacier StorageClassType = "GLACIER" +) + +// PermissionType defines permission type +type PermissionType string + +const ( + // PermissionRead permission type: READ + PermissionRead PermissionType = "READ" + + // PermissionWrite permission type: WRITE + PermissionWrite PermissionType = "WRITE" + + // PermissionReadAcp permission type: READ_ACP + PermissionReadAcp PermissionType = "READ_ACP" + + // PermissionWriteAcp permission type: WRITE_ACP + PermissionWriteAcp PermissionType = "WRITE_ACP" + + // PermissionFullControl permission type: FULL_CONTROL + PermissionFullControl PermissionType = "FULL_CONTROL" +) + +// GranteeType defines grantee type +type GranteeType string + +const ( + // GranteeGroup grantee type: Group + GranteeGroup GranteeType = "Group" + + // GranteeUser grantee type: CanonicalUser + GranteeUser GranteeType = "CanonicalUser" +) + +// GroupUriType defines grantee uri type +type GroupUriType string + +const ( + // GroupAllUsers grantee uri type: AllUsers + GroupAllUsers GroupUriType = "AllUsers" + + // GroupAuthenticatedUsers grantee uri type: AuthenticatedUsers + GroupAuthenticatedUsers GroupUriType = "AuthenticatedUsers" + + // GroupLogDelivery grantee uri type: LogDelivery + GroupLogDelivery GroupUriType = "LogDelivery" +) + +// VersioningStatusType defines bucket version status +type VersioningStatusType string + +const ( + // VersioningStatusEnabled version status: Enabled + VersioningStatusEnabled VersioningStatusType = "Enabled" + + // VersioningStatusSuspended version status: Suspended + VersioningStatusSuspended VersioningStatusType = "Suspended" +) + +// ProtocolType defines protocol type +type ProtocolType string + +const ( + // ProtocolHttp prorocol type: http + ProtocolHttp ProtocolType = "http" + + // ProtocolHttps prorocol type: https + ProtocolHttps ProtocolType = "https" +) + +// RuleStatusType defines lifeCycle rule status +type RuleStatusType string + +const ( + // RuleStatusEnabled rule status: Enabled + RuleStatusEnabled RuleStatusType = "Enabled" + + // RuleStatusDisabled rule status: Disabled + RuleStatusDisabled RuleStatusType = "Disabled" +) + +// RestoreTierType defines restore options +type RestoreTierType string + +const ( + // RestoreTierExpedited restore options: Expedited + RestoreTierExpedited RestoreTierType = "Expedited" + + // RestoreTierStandard restore options: Standard + RestoreTierStandard RestoreTierType = "Standard" + + // RestoreTierBulk restore options: Bulk + RestoreTierBulk RestoreTierType = "Bulk" +) + +// MetadataDirectiveType defines metadata operation indicator +type MetadataDirectiveType string + +const ( + // CopyMetadata metadata operation: COPY + CopyMetadata MetadataDirectiveType = "COPY" + + // ReplaceNew metadata operation: REPLACE_NEW + ReplaceNew MetadataDirectiveType = "REPLACE_NEW" + + // ReplaceMetadata metadata operation: REPLACE + ReplaceMetadata MetadataDirectiveType = "REPLACE" +) + +// EventType defines bucket notification type of events +type EventType string + +const ( + // ObjectCreatedAll type of events: ObjectCreated:* + ObjectCreatedAll EventType = "ObjectCreated:*" + + // ObjectCreatedPut type of events: ObjectCreated:Put + ObjectCreatedPut EventType = "ObjectCreated:Put" + + // ObjectCreatedPost type of events: ObjectCreated:Post + ObjectCreatedPost EventType = "ObjectCreated:Post" + + // ObjectCreatedCopy type of events: ObjectCreated:Copy + ObjectCreatedCopy EventType = "ObjectCreated:Copy" + + // ObjectCreatedCompleteMultipartUpload type of events: ObjectCreated:CompleteMultipartUpload + ObjectCreatedCompleteMultipartUpload EventType = "ObjectCreated:CompleteMultipartUpload" + + // ObjectRemovedAll type of events: ObjectRemoved:* + ObjectRemovedAll EventType = "ObjectRemoved:*" + + // ObjectRemovedDelete type of events: ObjectRemoved:Delete + ObjectRemovedDelete EventType = "ObjectRemoved:Delete" + + // ObjectRemovedDeleteMarkerCreated type of events: ObjectRemoved:DeleteMarkerCreated + ObjectRemovedDeleteMarkerCreated EventType = "ObjectRemoved:DeleteMarkerCreated" +) + +// PayerType defines type of payer +type PayerType string + +const ( + // BucketOwnerPayer type of payer: BucketOwner + BucketOwnerPayer PayerType = "BucketOwner" + + // RequesterPayer type of payer: Requester + RequesterPayer PayerType = "Requester" + + // Requester header for requester-Pays + Requester PayerType = "requester" +) + +// FetchPolicyStatusType defines type of fetch policy status +type FetchPolicyStatusType string + +const ( + // FetchStatusOpen type of status: open + FetchStatusOpen FetchPolicyStatusType = "open" + + // FetchStatusClosed type of status: closed + FetchStatusClosed FetchPolicyStatusType = "closed" +) + +// AvailableZoneType defines type of az redundancy +type AvailableZoneType string + +const ( + AvailableZoneMultiAz AvailableZoneType = "3az" +) + +// FSStatusType defines type of file system status +type FSStatusType string + +const ( + FSStatusEnabled FSStatusType = "Enabled" + FSStatusDisabled FSStatusType = "Disabled" +) + +// BucketType defines type of bucket +type BucketType string + +const ( + OBJECT BucketType = "OBJECT" + POSIX BucketType = "POSIX" +) diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/convert.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/convert.go new file mode 100644 index 0000000000..06b6762568 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/convert.go @@ -0,0 +1,1114 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "strconv" + "strings" + "time" +) + +func cleanHeaderPrefix(header http.Header) map[string][]string { + responseHeaders := make(map[string][]string) + for key, value := range header { + if len(value) > 0 { + key = strings.ToLower(key) + if strings.HasPrefix(key, HEADER_PREFIX) || strings.HasPrefix(key, HEADER_PREFIX_OBS) { + key = key[len(HEADER_PREFIX):] + } + responseHeaders[key] = value + } + } + return responseHeaders +} + +// ParseStringToEventType converts string value to EventType value and returns it +func ParseStringToEventType(value string) (ret EventType) { + switch value { + case "ObjectCreated:*", "s3:ObjectCreated:*": + ret = ObjectCreatedAll + case "ObjectCreated:Put", "s3:ObjectCreated:Put": + ret = ObjectCreatedPut + case "ObjectCreated:Post", "s3:ObjectCreated:Post": + ret = ObjectCreatedPost + case "ObjectCreated:Copy", "s3:ObjectCreated:Copy": + ret = ObjectCreatedCopy + case "ObjectCreated:CompleteMultipartUpload", "s3:ObjectCreated:CompleteMultipartUpload": + ret = ObjectCreatedCompleteMultipartUpload + case "ObjectRemoved:*", "s3:ObjectRemoved:*": + ret = ObjectRemovedAll + case "ObjectRemoved:Delete", "s3:ObjectRemoved:Delete": + ret = ObjectRemovedDelete + case "ObjectRemoved:DeleteMarkerCreated", "s3:ObjectRemoved:DeleteMarkerCreated": + ret = ObjectRemovedDeleteMarkerCreated + default: + ret = "" + } + return +} + +// ParseStringToStorageClassType converts string value to StorageClassType value and returns it +func ParseStringToStorageClassType(value string) (ret StorageClassType) { + switch value { + case "STANDARD": + ret = StorageClassStandard + case "STANDARD_IA", "WARM": + ret = StorageClassWarm + case "GLACIER", "COLD": + ret = StorageClassCold + default: + ret = "" + } + return +} + +func prepareGrantURI(grant Grant) string { + if grant.Grantee.URI == GroupAllUsers || grant.Grantee.URI == GroupAuthenticatedUsers { + return fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/global/", grant.Grantee.URI) + } + if grant.Grantee.URI == GroupLogDelivery { + return fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/s3/", grant.Grantee.URI) + } + return fmt.Sprintf("%s", grant.Grantee.URI) +} + +func convertGrantToXML(grant Grant, isObs bool, isBucket bool) string { + xml := make([]string, 0, 4) + + if grant.Grantee.Type == GranteeUser { + if isObs { + xml = append(xml, "") + } else { + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + } + if grant.Grantee.ID != "" { + granteeID := XmlTranscoding(grant.Grantee.ID) + xml = append(xml, fmt.Sprintf("%s", granteeID)) + } + if !isObs && grant.Grantee.DisplayName != "" { + granteeDisplayName := XmlTranscoding(grant.Grantee.DisplayName) + xml = append(xml, fmt.Sprintf("%s", granteeDisplayName)) + } + xml = append(xml, "") + } else { + if !isObs { + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + xml = append(xml, prepareGrantURI(grant)) + xml = append(xml, "") + } else if grant.Grantee.URI == GroupAllUsers { + xml = append(xml, "") + xml = append(xml, fmt.Sprintf("Everyone")) + xml = append(xml, "") + } else { + return strings.Join(xml, "") + } + } + + xml = append(xml, fmt.Sprintf("%s", grant.Permission)) + if isObs && isBucket { + xml = append(xml, fmt.Sprintf("%t", grant.Delivered)) + } + xml = append(xml, fmt.Sprintf("")) + return strings.Join(xml, "") +} + +func hasLoggingTarget(input BucketLoggingStatus) bool { + if input.TargetBucket != "" || input.TargetPrefix != "" || len(input.TargetGrants) > 0 { + return true + } + return false +} + +// ConvertLoggingStatusToXml converts BucketLoggingStatus value to XML data and returns it +func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool, isObs bool) (data string, md5 string) { + grantsLength := len(input.TargetGrants) + xml := make([]string, 0, 8+grantsLength) + + xml = append(xml, "") + if isObs && input.Agency != "" { + agency := XmlTranscoding(input.Agency) + xml = append(xml, fmt.Sprintf("%s", agency)) + } + if hasLoggingTarget(input) { + xml = append(xml, "") + if input.TargetBucket != "" { + xml = append(xml, fmt.Sprintf("%s", input.TargetBucket)) + } + if input.TargetPrefix != "" { + targetPrefix := XmlTranscoding(input.TargetPrefix) + xml = append(xml, fmt.Sprintf("%s", targetPrefix)) + } + if grantsLength > 0 { + xml = append(xml, "") + for _, grant := range input.TargetGrants { + xml = append(xml, convertGrantToXML(grant, isObs, false)) + } + xml = append(xml, "") + } + + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +// ConvertAclToXml converts AccessControlPolicy value to XML data and returns it +func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Grants)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if !isObs && input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + if isObs && input.Delivered != "" { + objectDelivered := XmlTranscoding(input.Delivered) + xml = append(xml, fmt.Sprintf("%s", objectDelivered)) + } else { + xml = append(xml, "") + } + for _, grant := range input.Grants { + xml = append(xml, convertGrantToXML(grant, isObs, false)) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertBucketACLToXML(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Grants)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if !isObs && input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + + xml = append(xml, "") + + for _, grant := range input.Grants { + xml = append(xml, convertGrantToXML(grant, isObs, true)) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertConditionToXML(condition Condition) string { + xml := make([]string, 0, 2) + if condition.KeyPrefixEquals != "" { + keyPrefixEquals := XmlTranscoding(condition.KeyPrefixEquals) + xml = append(xml, fmt.Sprintf("%s", keyPrefixEquals)) + } + if condition.HttpErrorCodeReturnedEquals != "" { + xml = append(xml, fmt.Sprintf("%s", condition.HttpErrorCodeReturnedEquals)) + } + if len(xml) > 0 { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + return "" +} + +func prepareRoutingRule(input BucketWebsiteConfiguration) string { + xml := make([]string, 0, len(input.RoutingRules)*10) + for _, routingRule := range input.RoutingRules { + xml = append(xml, "") + xml = append(xml, "") + if routingRule.Redirect.Protocol != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.Protocol)) + } + if routingRule.Redirect.HostName != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HostName)) + } + if routingRule.Redirect.ReplaceKeyPrefixWith != "" { + replaceKeyPrefixWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyPrefixWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyPrefixWith)) + } + + if routingRule.Redirect.ReplaceKeyWith != "" { + replaceKeyWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyWith)) + } + if routingRule.Redirect.HttpRedirectCode != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HttpRedirectCode)) + } + xml = append(xml, "") + + if ret := convertConditionToXML(routingRule.Condition); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, "") + } + return strings.Join(xml, "") +} + +// ConvertWebsiteConfigurationToXml converts BucketWebsiteConfiguration value to XML data and returns it +func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd5 bool) (data string, md5 string) { + routingRuleLength := len(input.RoutingRules) + xml := make([]string, 0, 6+routingRuleLength*10) + xml = append(xml, "") + + if input.RedirectAllRequestsTo.HostName != "" { + xml = append(xml, fmt.Sprintf("%s", input.RedirectAllRequestsTo.HostName)) + if input.RedirectAllRequestsTo.Protocol != "" { + xml = append(xml, fmt.Sprintf("%s", input.RedirectAllRequestsTo.Protocol)) + } + xml = append(xml, "") + } else { + if input.IndexDocument.Suffix != "" { + indexDocumentSuffix := XmlTranscoding(input.IndexDocument.Suffix) + xml = append(xml, fmt.Sprintf("%s", indexDocumentSuffix)) + } + if input.ErrorDocument.Key != "" { + errorDocumentKey := XmlTranscoding(input.ErrorDocument.Key) + xml = append(xml, fmt.Sprintf("%s", errorDocumentKey)) + } + if routingRuleLength > 0 { + xml = append(xml, "") + xml = append(xml, prepareRoutingRule(input)) + xml = append(xml, "") + } + } + + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertTransitionsToXML(transitions []Transition, isObs bool) string { + if length := len(transitions); length > 0 { + xml := make([]string, 0, length) + for _, transition := range transitions { + var temp string + if transition.Days > 0 { + temp = fmt.Sprintf("%d", transition.Days) + } else if !transition.Date.IsZero() { + temp = fmt.Sprintf("%s", transition.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT)) + } + if temp != "" { + if !isObs { + storageClass := string(transition.StorageClass) + if transition.StorageClass == StorageClassWarm { + storageClass = string(storageClassStandardIA) + } else if transition.StorageClass == StorageClassCold { + storageClass = string(storageClassGlacier) + } + xml = append(xml, fmt.Sprintf("%s%s", temp, storageClass)) + } else { + xml = append(xml, fmt.Sprintf("%s%s", temp, transition.StorageClass)) + } + } + } + return strings.Join(xml, "") + } + return "" +} + +func convertExpirationToXML(expiration Expiration) string { + if expiration.Days > 0 { + return fmt.Sprintf("%d", expiration.Days) + } else if !expiration.Date.IsZero() { + return fmt.Sprintf("%s", expiration.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT)) + } + return "" +} +func convertNoncurrentVersionTransitionsToXML(noncurrentVersionTransitions []NoncurrentVersionTransition, isObs bool) string { + if length := len(noncurrentVersionTransitions); length > 0 { + xml := make([]string, 0, length) + for _, noncurrentVersionTransition := range noncurrentVersionTransitions { + if noncurrentVersionTransition.NoncurrentDays > 0 { + storageClass := string(noncurrentVersionTransition.StorageClass) + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + xml = append(xml, fmt.Sprintf("%d"+ + "%s", + noncurrentVersionTransition.NoncurrentDays, storageClass)) + } + } + return strings.Join(xml, "") + } + return "" +} +func convertNoncurrentVersionExpirationToXML(noncurrentVersionExpiration NoncurrentVersionExpiration) string { + if noncurrentVersionExpiration.NoncurrentDays > 0 { + return fmt.Sprintf("%d", noncurrentVersionExpiration.NoncurrentDays) + } + return "" +} + +// ConvertLifecyleConfigurationToXml converts BucketLifecyleConfiguration value to XML data and returns it +func ConvertLifecyleConfigurationToXml(input BucketLifecyleConfiguration, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.LifecycleRules)*9) + xml = append(xml, "") + for _, lifecyleRule := range input.LifecycleRules { + xml = append(xml, "") + if lifecyleRule.ID != "" { + lifecyleRuleID := XmlTranscoding(lifecyleRule.ID) + xml = append(xml, fmt.Sprintf("%s", lifecyleRuleID)) + } + lifecyleRulePrefix := XmlTranscoding(lifecyleRule.Prefix) + xml = append(xml, fmt.Sprintf("%s", lifecyleRulePrefix)) + xml = append(xml, fmt.Sprintf("%s", lifecyleRule.Status)) + if ret := convertTransitionsToXML(lifecyleRule.Transitions, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := convertExpirationToXML(lifecyleRule.Expiration); ret != "" { + xml = append(xml, ret) + } + if ret := convertNoncurrentVersionTransitionsToXML(lifecyleRule.NoncurrentVersionTransitions, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := convertNoncurrentVersionExpirationToXML(lifecyleRule.NoncurrentVersionExpiration); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +// ConvertEncryptionConfigurationToXml converts BucketEncryptionConfiguration value to XML data and returns it +func ConvertEncryptionConfigurationToXml(input BucketEncryptionConfiguration, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 5) + xml = append(xml, "") + + algorithm := XmlTranscoding(input.SSEAlgorithm) + xml = append(xml, fmt.Sprintf("%s", algorithm)) + + if input.KMSMasterKeyID != "" { + kmsKeyID := XmlTranscoding(input.KMSMasterKeyID) + xml = append(xml, fmt.Sprintf("%s", kmsKeyID)) + } + if input.ProjectID != "" { + projectID := XmlTranscoding(input.ProjectID) + xml = append(xml, fmt.Sprintf("%s", projectID)) + } + + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func converntFilterRulesToXML(filterRules []FilterRule, isObs bool) string { + if length := len(filterRules); length > 0 { + xml := make([]string, 0, length*4) + for _, filterRule := range filterRules { + xml = append(xml, "") + if filterRule.Name != "" { + filterRuleName := XmlTranscoding(filterRule.Name) + xml = append(xml, fmt.Sprintf("%s", filterRuleName)) + } + if filterRule.Value != "" { + filterRuleValue := XmlTranscoding(filterRule.Value) + xml = append(xml, fmt.Sprintf("%s", filterRuleValue)) + } + xml = append(xml, "") + } + if !isObs { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + return "" +} + +func converntEventsToXML(events []EventType, isObs bool) string { + if length := len(events); length > 0 { + xml := make([]string, 0, length) + if !isObs { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s%s", "s3:", event)) + } + } else { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s", event)) + } + } + return strings.Join(xml, "") + } + return "" +} + +func converntConfigureToXML(topicConfiguration TopicConfiguration, xmlElem string, isObs bool) string { + xml := make([]string, 0, 6) + xml = append(xml, xmlElem) + if topicConfiguration.ID != "" { + topicConfigurationID := XmlTranscoding(topicConfiguration.ID) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationID)) + } + topicConfigurationTopic := XmlTranscoding(topicConfiguration.Topic) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationTopic)) + + if ret := converntEventsToXML(topicConfiguration.Events, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := converntFilterRulesToXML(topicConfiguration.FilterRules, isObs); ret != "" { + xml = append(xml, ret) + } + tempElem := xmlElem[0:1] + "/" + xmlElem[1:] + xml = append(xml, tempElem) + return strings.Join(xml, "") +} + +// ConverntObsRestoreToXml converts RestoreObjectInput value to XML data and returns it +func ConverntObsRestoreToXml(restoreObjectInput RestoreObjectInput) string { + xml := make([]string, 0, 2) + xml = append(xml, fmt.Sprintf("%d", restoreObjectInput.Days)) + if restoreObjectInput.Tier != "Bulk" { + xml = append(xml, fmt.Sprintf("%s", restoreObjectInput.Tier)) + } + xml = append(xml, fmt.Sprintf("")) + data := strings.Join(xml, "") + return data +} + +// ConvertNotificationToXml converts BucketNotification value to XML data and returns it +func ConvertNotificationToXml(input BucketNotification, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.TopicConfigurations)*6) + xml = append(xml, "") + for _, topicConfiguration := range input.TopicConfigurations { + ret := converntConfigureToXML(topicConfiguration, "", isObs) + xml = append(xml, ret) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +// ConvertCompleteMultipartUploadInputToXml converts CompleteMultipartUploadInput value to XML data and returns it +func ConvertCompleteMultipartUploadInputToXml(input CompleteMultipartUploadInput, returnMd5 bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.Parts)*4) + xml = append(xml, "") + for _, part := range input.Parts { + xml = append(xml, "") + xml = append(xml, fmt.Sprintf("%d", part.PartNumber)) + xml = append(xml, fmt.Sprintf("%s", part.ETag)) + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertDeleteObjectsToXML(input DeleteObjectsInput) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Objects)*4) + xml = append(xml, "") + if input.Quiet { + xml = append(xml, fmt.Sprintf("%t", input.Quiet)) + } + if input.EncodingType != "" { + encodingType := XmlTranscoding(input.EncodingType) + xml = append(xml, fmt.Sprintf("%s", encodingType)) + } + for _, obj := range input.Objects { + xml = append(xml, "") + key := XmlTranscoding(obj.Key) + xml = append(xml, fmt.Sprintf("%s", key)) + if obj.VersionId != "" { + xml = append(xml, fmt.Sprintf("%s", obj.VersionId)) + } + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + md5 = Base64Md5([]byte(data)) + return +} + +func parseSseHeader(responseHeaders map[string][]string) (sseHeader ISseHeader) { + if ret, ok := responseHeaders[HEADER_SSEC_ENCRYPTION]; ok { + sseCHeader := SseCHeader{Encryption: ret[0]} + if ret, ok = responseHeaders[HEADER_SSEC_KEY_MD5]; ok { + sseCHeader.KeyMD5 = ret[0] + } + sseHeader = sseCHeader + } else if ret, ok := responseHeaders[HEADER_SSEKMS_ENCRYPTION]; ok { + sseKmsHeader := SseKmsHeader{Encryption: ret[0]} + if ret, ok = responseHeaders[HEADER_SSEKMS_KEY]; ok { + sseKmsHeader.Key = ret[0] + } else if ret, ok = responseHeaders[HEADER_SSEKMS_ENCRYPT_KEY_OBS]; ok { + sseKmsHeader.Key = ret[0] + } + sseHeader = sseKmsHeader + } + return +} + +func parseCorsHeader(output BaseModel) (AllowOrigin, AllowHeader, AllowMethod, ExposeHeader string, MaxAgeSeconds int) { + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { + AllowOrigin = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok { + AllowHeader = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok { + MaxAgeSeconds = StringToInt(ret[0], 0) + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok { + AllowMethod = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { + ExposeHeader = ret[0] + } + return +} + +func parseUnCommonHeader(output *GetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok { + output.WebsiteRedirectLocation = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EXPIRATION]; ok { + output.Expiration = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_RESTORE]; ok { + output.Restore = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_OBJECT_TYPE]; ok { + output.ObjectType = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_NEXT_APPEND_POSITION]; ok { + output.NextAppendPosition = ret[0] + } +} + +// ParseGetObjectMetadataOutput sets GetObjectMetadataOutput field values with response headers +func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { + output.AllowOrigin, output.AllowHeader, output.AllowMethod, output.ExposeHeader, output.MaxAgeSeconds = parseCorsHeader(output.BaseModel) + parseUnCommonHeader(output) + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { + output.ContentType = ret[0] + } + + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_LASTMODIFIED]; ok { + ret, err := time.Parse(time.RFC1123, ret[0]) + if err == nil { + output.LastModified = ret + } + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LENGTH]; ok { + output.ContentLength = StringToInt64(ret[0], 0) + } + + output.Metadata = make(map[string]string) + + for key, value := range output.ResponseHeaders { + if strings.HasPrefix(key, PREFIX_META) { + _key := key[len(PREFIX_META):] + output.ResponseHeaders[_key] = value + output.Metadata[_key] = value[0] + delete(output.ResponseHeaders, key) + } + } + +} + +// ParseCopyObjectOutput sets CopyObjectOutput field values with response headers +func ParseCopyObjectOutput(output *CopyObjectOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_COPY_SOURCE_VERSION_ID]; ok { + output.CopySourceVersionId = ret[0] + } +} + +// ParsePutObjectOutput sets PutObjectOutput field values with response headers +func ParsePutObjectOutput(output *PutObjectOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } +} + +// ParseInitiateMultipartUploadOutput sets InitiateMultipartUploadOutput field values with response headers +func ParseInitiateMultipartUploadOutput(output *InitiateMultipartUploadOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) +} + +// ParseUploadPartOutput sets UploadPartOutput field values with response headers +func ParseUploadPartOutput(output *UploadPartOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } +} + +// ParseCompleteMultipartUploadOutput sets CompleteMultipartUploadOutput field values with response headers +func ParseCompleteMultipartUploadOutput(output *CompleteMultipartUploadOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } +} + +// ParseCopyPartOutput sets CopyPartOutput field values with response headers +func ParseCopyPartOutput(output *CopyPartOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) +} + +// ParseGetBucketMetadataOutput sets GetBucketMetadataOutput field values with response headers +func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) { + output.AllowOrigin, output.AllowHeader, output.AllowMethod, output.ExposeHeader, output.MaxAgeSeconds = parseCorsHeader(output.BaseModel) + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_VERSION_OBS]; ok { + output.Version = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = ret[0] + } else if ret, ok := output.ResponseHeaders[HEADER_BUCKET_LOCATION_OBS]; ok { + output.Location = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EPID_HEADERS]; ok { + output.Epid = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_AZ_REDUNDANCY]; ok { + output.AZRedundancy = ret[0] + } + if ret, ok := output.ResponseHeaders[headerFSFileInterface]; ok { + output.FSStatus = parseStringToFSStatusType(ret[0]) + } else { + output.FSStatus = FSStatusDisabled + } +} + +func parseContentHeader(output *SetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok { + output.ContentDisposition = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok { + output.ContentEncoding = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok { + output.ContentLanguage = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { + output.ContentType = ret[0] + } +} + +// ParseSetObjectMetadataOutput sets SetObjectMetadataOutput field values with response headers +func ParseSetObjectMetadataOutput(output *SetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_METADATA_DIRECTIVE]; ok { + output.MetadataDirective = MetadataDirectiveType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok { + output.CacheControl = ret[0] + } + parseContentHeader(output) + if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok { + output.Expires = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok { + output.WebsiteRedirectLocation = ret[0] + } + output.Metadata = make(map[string]string) + + for key, value := range output.ResponseHeaders { + if strings.HasPrefix(key, PREFIX_META) { + _key := key[len(PREFIX_META):] + output.ResponseHeaders[_key] = value + output.Metadata[_key] = value[0] + delete(output.ResponseHeaders, key) + } + } +} + +// ParseDeleteObjectOutput sets DeleteObjectOutput field values with response headers +func ParseDeleteObjectOutput(output *DeleteObjectOutput) { + if versionID, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionID[0] + } + + if deleteMarker, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok { + output.DeleteMarker = deleteMarker[0] == "true" + } +} + +// ParseGetObjectOutput sets GetObjectOutput field values with response headers +func ParseGetObjectOutput(output *GetObjectOutput) { + ParseGetObjectMetadataOutput(&output.GetObjectMetadataOutput) + if ret, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok { + output.DeleteMarker = ret[0] == "true" + } + if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok { + output.CacheControl = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok { + output.ContentDisposition = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok { + output.ContentEncoding = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok { + output.ContentLanguage = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok { + output.Expires = ret[0] + } +} + +// ConvertRequestToIoReaderV2 converts req to XML data +func ConvertRequestToIoReaderV2(req interface{}) (io.Reader, string, error) { + data, err := TransToXml(req) + if err == nil { + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "Do http request with data: %s", string(data)) + } + return bytes.NewReader(data), Base64Md5(data), nil + } + return nil, "", err +} + +// ConvertRequestToIoReader converts req to XML data +func ConvertRequestToIoReader(req interface{}) (io.Reader, error) { + body, err := TransToXml(req) + if err == nil { + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "Do http request with data: %s", string(body)) + } + return bytes.NewReader(body), nil + } + return nil, err +} + +func parseBucketPolicyOutput(s reflect.Type, baseModel IBaseModel, body []byte) { + for i := 0; i < s.NumField(); i++ { + if s.Field(i).Tag == "json:\"body\"" { + reflect.ValueOf(baseModel).Elem().FieldByName(s.Field(i).Name).SetString(string(body)) + break + } + } +} + +// ParseResponseToBaseModel gets response from OBS +func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResult bool, isObs bool) (err error) { + readCloser, ok := baseModel.(IReadCloser) + if !ok { + defer func() { + errMsg := resp.Body.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close response body") + } + }() + var body []byte + body, err = ioutil.ReadAll(resp.Body) + if err == nil && len(body) > 0 { + if xmlResult { + err = ParseXml(body, baseModel) + } else { + s := reflect.TypeOf(baseModel).Elem() + if reflect.TypeOf(baseModel).Elem().Name() == "GetBucketPolicyOutput" { + parseBucketPolicyOutput(s, baseModel, body) + } else { + err = parseJSON(body, baseModel) + } + } + if err != nil { + doLog(LEVEL_ERROR, "Unmarshal error: %v", err) + } + } + } else { + readCloser.setReadCloser(resp.Body) + } + + baseModel.setStatusCode(resp.StatusCode) + responseHeaders := cleanHeaderPrefix(resp.Header) + baseModel.setResponseHeaders(responseHeaders) + if values, ok := responseHeaders[HEADER_REQUEST_ID]; ok { + baseModel.setRequestID(values[0]) + } + return +} + +// ParseResponseToObsError gets obsError from OBS +func ParseResponseToObsError(resp *http.Response, isObs bool) error { + isJson := false + if contentType, ok := resp.Header[HEADER_CONTENT_TYPE_CAML]; ok { + jsonType, _ := mimeTypes["json"] + isJson = contentType[0] == jsonType + } + obsError := ObsError{} + respError := ParseResponseToBaseModel(resp, &obsError, !isJson, isObs) + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + obsError.Status = resp.Status + return obsError +} + +// convertFetchPolicyToJSON converts SetBucketFetchPolicyInput into json format +func convertFetchPolicyToJSON(input SetBucketFetchPolicyInput) (data string, err error) { + fetch := map[string]SetBucketFetchPolicyInput{"fetch": input} + json, err := json.Marshal(fetch) + if err != nil { + return "", err + } + data = string(json) + return +} + +// convertFetchJobToJSON converts SetBucketFetchJobInput into json format +func convertFetchJobToJSON(input SetBucketFetchJobInput) (data string, err error) { + objectHeaders := make(map[string]string) + for key, value := range input.ObjectHeaders { + if value != "" { + _key := strings.ToLower(key) + if !strings.HasPrefix(key, HEADER_PREFIX_OBS) { + _key = HEADER_PREFIX_META_OBS + _key + } + objectHeaders[_key] = value + } + } + input.ObjectHeaders = objectHeaders + json, err := json.Marshal(input) + if err != nil { + return "", err + } + data = string(json) + return +} + +func parseStringToFSStatusType(value string) (ret FSStatusType) { + switch value { + case "Enabled": + ret = FSStatusEnabled + case "Disabled": + ret = FSStatusDisabled + default: + ret = "" + } + return +} + +func decodeListObjectsOutput(output *ListObjectsOutput) (err error) { + output.Delimiter, err = url.QueryUnescape(output.Delimiter) + if err != nil { + return + } + output.Marker, err = url.QueryUnescape(output.Marker) + if err != nil { + return + } + output.NextMarker, err = url.QueryUnescape(output.NextMarker) + if err != nil { + return + } + output.Prefix, err = url.QueryUnescape(output.Prefix) + if err != nil { + return + } + for index, value := range output.CommonPrefixes { + output.CommonPrefixes[index], err = url.QueryUnescape(value) + if err != nil { + return + } + } + for index, content := range output.Contents { + output.Contents[index].Key, err = url.QueryUnescape(content.Key) + if err != nil { + return + } + } + return +} + +func decodeListVersionsOutput(output *ListVersionsOutput) (err error) { + output.Delimiter, err = url.QueryUnescape(output.Delimiter) + if err != nil { + return + } + output.KeyMarker, err = url.QueryUnescape(output.KeyMarker) + if err != nil { + return + } + output.NextKeyMarker, err = url.QueryUnescape(output.NextKeyMarker) + if err != nil { + return + } + output.Prefix, err = url.QueryUnescape(output.Prefix) + if err != nil { + return + } + for index, version := range output.Versions { + output.Versions[index].Key, err = url.QueryUnescape(version.Key) + if err != nil { + return + } + } + for index, deleteMarker := range output.DeleteMarkers { + output.DeleteMarkers[index].Key, err = url.QueryUnescape(deleteMarker.Key) + if err != nil { + return + } + } + for index, value := range output.CommonPrefixes { + output.CommonPrefixes[index], err = url.QueryUnescape(value) + if err != nil { + return + } + } + return +} + +func decodeDeleteObjectsOutput(output *DeleteObjectsOutput) (err error) { + for index, object := range output.Deleteds { + output.Deleteds[index].Key, err = url.QueryUnescape(object.Key) + if err != nil { + return + } + } + for index, object := range output.Errors { + output.Errors[index].Key, err = url.QueryUnescape(object.Key) + if err != nil { + return + } + } + return +} + +func decodeListMultipartUploadsOutput(output *ListMultipartUploadsOutput) (err error) { + output.Delimiter, err = url.QueryUnescape(output.Delimiter) + if err != nil { + return + } + output.Prefix, err = url.QueryUnescape(output.Prefix) + if err != nil { + return + } + output.KeyMarker, err = url.QueryUnescape(output.KeyMarker) + if err != nil { + return + } + output.NextKeyMarker, err = url.QueryUnescape(output.NextKeyMarker) + if err != nil { + return + } + for index, value := range output.CommonPrefixes { + output.CommonPrefixes[index], err = url.QueryUnescape(value) + if err != nil { + return + } + } + for index, upload := range output.Uploads { + output.Uploads[index].Key, err = url.QueryUnescape(upload.Key) + if err != nil { + return + } + } + return +} + +func decodeListPartsOutput(output *ListPartsOutput) (err error) { + output.Key, err = url.QueryUnescape(output.Key) + return +} + +func decodeInitiateMultipartUploadOutput(output *InitiateMultipartUploadOutput) (err error) { + output.Key, err = url.QueryUnescape(output.Key) + return +} + +func decodeCompleteMultipartUploadOutput(output *CompleteMultipartUploadOutput) (err error) { + output.Key, err = url.QueryUnescape(output.Key) + return +} + +// ParseAppendObjectOutput sets AppendObjectOutput field values with response headers +func ParseAppendObjectOutput(output *AppendObjectOutput) (err error) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_NEXT_APPEND_POSITION]; ok { + output.NextAppendPosition, err = strconv.ParseInt(ret[0], 10, 64) + if err != nil { + err = fmt.Errorf("failed to parse next append position with error [%v]", err) + } + } + return +} + +// ParseModifyObjectOutput sets ModifyObjectOutput field values with response headers +func ParseModifyObjectOutput(output *ModifyObjectOutput) { + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/error.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/error.go new file mode 100644 index 0000000000..eb8ab28099 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/error.go @@ -0,0 +1,34 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "encoding/xml" + "fmt" +) + +// ObsError defines error response from OBS +type ObsError struct { + BaseModel + Status string + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code" json:"code"` + Message string `xml:"Message" json:"message"` + Resource string `xml:"Resource"` + HostId string `xml:"HostId"` +} + +func (err ObsError) Error() string { + return fmt.Sprintf("obs: service returned error: Status=%s, Code=%s, Message=%s, RequestId=%s", + err.Status, err.Code, err.Message, err.RequestId) +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/extension.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/extension.go new file mode 100644 index 0000000000..7fa811628e --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/extension.go @@ -0,0 +1,41 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "fmt" + "strconv" + "strings" +) + +type extensionOptions interface{} +type extensionHeaders func(headers map[string][]string, isObs bool) error + +func setHeaderPrefix(key string, value string) extensionHeaders { + return func(headers map[string][]string, isObs bool) error { + if strings.TrimSpace(value) == "" { + return fmt.Errorf("set header %s with empty value", key) + } + setHeaders(headers, key, []string{value}, isObs) + return nil + } +} + +// WithReqPaymentHeader sets header for requester-pays +func WithReqPaymentHeader(requester PayerType) extensionHeaders { + return setHeaderPrefix(REQUEST_PAYER, string(requester)) +} + +func WithTrafficLimitHeader(trafficLimit int64) extensionHeaders { + return setHeaderPrefix(TRAFFIC_LIMIT, strconv.FormatInt(trafficLimit, 10)) +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/http.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/http.go new file mode 100644 index 0000000000..4464c722ef --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/http.go @@ -0,0 +1,637 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "bytes" + "errors" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +func prepareHeaders(headers map[string][]string, meta bool, isObs bool) map[string][]string { + _headers := make(map[string][]string, len(headers)) + if headers != nil { + for key, value := range headers { + key = strings.TrimSpace(key) + if key == "" { + continue + } + _key := strings.ToLower(key) + if _, ok := allowedRequestHTTPHeaderMetadataNames[_key]; !ok && !strings.HasPrefix(key, HEADER_PREFIX) && !strings.HasPrefix(key, HEADER_PREFIX_OBS) { + if !meta { + continue + } + if !isObs { + _key = HEADER_PREFIX_META + _key + } else { + _key = HEADER_PREFIX_META_OBS + _key + } + } else { + _key = key + } + _headers[_key] = value + } + } + return _headers +} + +func (obsClient ObsClient) doActionWithoutBucket(action, method string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + return obsClient.doAction(action, method, "", "", input, output, true, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketV2(action, method, bucketName string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + return obsClient.doAction(action, method, bucketName, "", input, output, false, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucket(action, method, bucketName string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + return obsClient.doAction(action, method, bucketName, "", input, output, true, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketAndKeyV2(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + if strings.TrimSpace(objectKey) == "" { + return errors.New("Key is empty") + } + return obsClient.doAction(action, method, bucketName, objectKey, input, output, false, true, extensions) +} + +func (obsClient ObsClient) doActionWithBucketAndKeyUnRepeatable(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, extensions []extensionOptions) error { + return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, false, extensions) +} + +func (obsClient ObsClient) _doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, repeatable bool, extensions []extensionOptions) error { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") + } + if strings.TrimSpace(objectKey) == "" { + return errors.New("Key is empty") + } + return obsClient.doAction(action, method, bucketName, objectKey, input, output, true, repeatable, extensions) +} + +func (obsClient ObsClient) doAction(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, xmlResult bool, repeatable bool, extensions []extensionOptions) error { + + var resp *http.Response + var respError error + doLog(LEVEL_INFO, "Enter method %s...", action) + start := GetCurrentTimestamp() + + params, headers, data, err := input.trans(obsClient.conf.signature == SignatureObs) + if err != nil { + return err + } + + if params == nil { + params = make(map[string]string) + } + + if headers == nil { + headers = make(map[string][]string) + } + + for _, extension := range extensions { + if extensionHeader, ok := extension.(extensionHeaders); ok { + _err := extensionHeader(headers, obsClient.conf.signature == SignatureObs) + if _err != nil { + doLog(LEVEL_INFO, fmt.Sprintf("set header with error: %v", _err)) + } + } else { + doLog(LEVEL_INFO, "Unsupported extensionOptions") + } + } + + switch method { + case HTTP_GET: + resp, respError = obsClient.doHTTPGet(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_POST: + resp, respError = obsClient.doHTTPPost(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_PUT: + resp, respError = obsClient.doHTTPPut(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_DELETE: + resp, respError = obsClient.doHTTPDelete(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_HEAD: + resp, respError = obsClient.doHTTPHead(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_OPTIONS: + resp, respError = obsClient.doHTTPOptions(bucketName, objectKey, params, headers, data, repeatable) + default: + respError = errors.New("Unexpect http method error") + } + if respError == nil && output != nil { + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + } else { + doLog(LEVEL_WARN, "Do http request with error: %v", respError) + } + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } + + return respError +} + +func (obsClient ObsClient) doHTTPGet(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_GET, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPHead(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_HEAD, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPOptions(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_OPTIONS, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPDelete(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_DELETE, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPPut(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_PUT, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHTTPPost(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHTTP(HTTP_POST, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func prepareAgentHeader(clientUserAgent string) string { + userAgent := USER_AGENT + if clientUserAgent != "" { + userAgent = clientUserAgent + } + return userAgent +} + +func (obsClient ObsClient) getSignedURLResponse(action string, output IBaseModel, xmlResult bool, resp *http.Response, err error, start int64) (respError error) { + var msg interface{} + if err != nil { + respError = err + resp = nil + } else { + doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) + if resp.StatusCode >= 300 { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + msg = resp.Status + resp = nil + } else { + if output != nil { + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) + } + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + } + } + + if msg != nil { + doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) + } + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } + return +} + +func (obsClient ObsClient) doHTTPWithSignedURL(action, method string, signedURL string, actualSignedRequestHeaders http.Header, data io.Reader, output IBaseModel, xmlResult bool) (respError error) { + req, err := http.NewRequest(method, signedURL, data) + if err != nil { + return err + } + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } + var resp *http.Response + + var isSecurityToken bool + var securityToken string + var query []string + parmas := strings.Split(signedURL, "?") + if len(parmas) > 1 { + query = strings.Split(parmas[1], "&") + for _, value := range query { + if strings.HasPrefix(value, HEADER_STS_TOKEN_AMZ+"=") || strings.HasPrefix(value, HEADER_STS_TOKEN_OBS+"=") { + if value[len(HEADER_STS_TOKEN_AMZ)+1:] != "" { + securityToken = value[len(HEADER_STS_TOKEN_AMZ)+1:] + isSecurityToken = true + } + } + } + } + logSignedURL := signedURL + if isSecurityToken { + logSignedURL = strings.Replace(logSignedURL, securityToken, "******", -1) + } + doLog(LEVEL_INFO, "Do %s with signedUrl %s...", action, logSignedURL) + + req.Header = actualSignedRequestHeaders + if value, ok := req.Header[HEADER_HOST_CAMEL]; ok { + req.Host = value[0] + delete(req.Header, HEADER_HOST_CAMEL) + } else if value, ok := req.Header[HEADER_HOST]; ok { + req.Host = value[0] + delete(req.Header, HEADER_HOST) + } + + if value, ok := req.Header[HEADER_CONTENT_LENGTH_CAMEL]; ok { + req.ContentLength = StringToInt64(value[0], -1) + delete(req.Header, HEADER_CONTENT_LENGTH_CAMEL) + } else if value, ok := req.Header[HEADER_CONTENT_LENGTH]; ok { + req.ContentLength = StringToInt64(value[0], -1) + delete(req.Header, HEADER_CONTENT_LENGTH) + } + + userAgent := prepareAgentHeader(obsClient.conf.userAgent) + req.Header[HEADER_USER_AGENT_CAMEL] = []string{userAgent} + start := GetCurrentTimestamp() + resp, err = obsClient.httpClient.Do(req) + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + } + + respError = obsClient.getSignedURLResponse(action, output, xmlResult, resp, err, start) + + return +} + +func prepareData(headers map[string][]string, data interface{}) (io.Reader, error) { + var _data io.Reader + if data != nil { + if dataStr, ok := data.(string); ok { + doLog(LEVEL_DEBUG, "Do http request with string: %s", dataStr) + headers["Content-Length"] = []string{IntToString(len(dataStr))} + _data = strings.NewReader(dataStr) + } else if dataByte, ok := data.([]byte); ok { + doLog(LEVEL_DEBUG, "Do http request with byte array") + headers["Content-Length"] = []string{IntToString(len(dataByte))} + _data = bytes.NewReader(dataByte) + } else if dataReader, ok := data.(io.Reader); ok { + _data = dataReader + } else { + doLog(LEVEL_WARN, "Data is not a valid io.Reader") + return nil, errors.New("Data is not a valid io.Reader") + } + } + return _data, nil +} + +func (obsClient ObsClient) getRequest(redirectURL, requestURL string, redirectFlag bool, _data io.Reader, method, + bucketName, objectKey string, params map[string]string, headers map[string][]string) (*http.Request, error) { + if redirectURL != "" { + if !redirectFlag { + parsedRedirectURL, err := url.Parse(redirectURL) + if err != nil { + return nil, err + } + requestURL, err = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectURL.Host) + if err != nil { + return nil, err + } + if parsedRequestURL, err := url.Parse(requestURL); err != nil { + return nil, err + } else if parsedRequestURL.RawQuery != "" && parsedRedirectURL.RawQuery == "" { + redirectURL += "?" + parsedRequestURL.RawQuery + } + } + requestURL = redirectURL + } else { + var err error + requestURL, err = obsClient.doAuth(method, bucketName, objectKey, params, headers, "") + if err != nil { + return nil, err + } + } + + req, err := http.NewRequest(method, requestURL, _data) + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } + if err != nil { + return nil, err + } + doLog(LEVEL_DEBUG, "Do request with url [%s] and method [%s]", requestURL, method) + return req, nil +} + +func logHeaders(headers map[string][]string, signature SignatureType) { + if isDebugLogEnabled() { + auth := headers[HEADER_AUTH_CAMEL] + delete(headers, HEADER_AUTH_CAMEL) + + var isSecurityToken bool + var securityToken []string + if securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_AMZ]; isSecurityToken { + headers[HEADER_STS_TOKEN_AMZ] = []string{"******"} + } else if securityToken, isSecurityToken = headers[HEADER_STS_TOKEN_OBS]; isSecurityToken { + headers[HEADER_STS_TOKEN_OBS] = []string{"******"} + } + doLog(LEVEL_DEBUG, "Request headers: %v", headers) + headers[HEADER_AUTH_CAMEL] = auth + if isSecurityToken { + if signature == SignatureObs { + headers[HEADER_STS_TOKEN_OBS] = securityToken + } else { + headers[HEADER_STS_TOKEN_AMZ] = securityToken + } + } + } +} + +func prepareReq(headers map[string][]string, req, lastRequest *http.Request, clientUserAgent string) *http.Request { + for key, value := range headers { + if key == HEADER_HOST_CAMEL { + req.Host = value[0] + delete(headers, key) + } else if key == HEADER_CONTENT_LENGTH_CAMEL { + req.ContentLength = StringToInt64(value[0], -1) + delete(headers, key) + } else { + req.Header[key] = value + } + } + + lastRequest = req + + userAgent := prepareAgentHeader(clientUserAgent) + req.Header[HEADER_USER_AGENT_CAMEL] = []string{userAgent} + + if lastRequest != nil { + req.Host = lastRequest.Host + req.ContentLength = lastRequest.ContentLength + } + return lastRequest +} + +func canNotRetry(repeatable bool, statusCode int) bool { + if !repeatable || (statusCode >= 400 && statusCode < 500) || statusCode == 304 { + return true + } + return false +} + +func isRedirectErr(location string, redirectCount, maxRedirectCount int) bool { + if location != "" && redirectCount < maxRedirectCount { + return true + } + return false +} + +func setRedirectFlag(statusCode int, method string) (redirectFlag bool) { + if statusCode == 302 && method == HTTP_GET { + redirectFlag = true + } else { + redirectFlag = false + } + return +} + +func prepareRetry(resp *http.Response, headers map[string][]string, _data io.Reader, msg interface{}) (io.Reader, *http.Response, error) { + if resp != nil { + _err := resp.Body.Close() + checkAndLogErr(_err, LEVEL_WARN, "Failed to close resp body") + resp = nil + } + if _, ok := headers[HEADER_AUTH_CAMEL]; ok { + delete(headers, HEADER_AUTH_CAMEL) + } + doLog(LEVEL_WARN, "Failed to send request with reason:%v, will try again", msg) + if r, ok := _data.(*strings.Reader); ok { + _, err := r.Seek(0, 0) + if err != nil { + return nil, nil, err + } + } else if r, ok := _data.(*bytes.Reader); ok { + _, err := r.Seek(0, 0) + if err != nil { + return nil, nil, err + } + } else if r, ok := _data.(*fileReaderWrapper); ok { + fd, err := os.Open(r.filePath) + if err != nil { + return nil, nil, err + } + fileReaderWrapper := &fileReaderWrapper{filePath: r.filePath} + fileReaderWrapper.mark = r.mark + fileReaderWrapper.reader = fd + fileReaderWrapper.totalCount = r.totalCount + _data = fileReaderWrapper + _, err = fd.Seek(r.mark, 0) + if err != nil { + errMsg := fd.Close() + checkAndLogErr(errMsg, LEVEL_WARN, "Failed to close with reason: %v", errMsg) + return nil, nil, err + } + } else if r, ok := _data.(*readerWrapper); ok { + _, err := r.seek(0, 0) + if err != nil { + return nil, nil, err + } + } + return _data, resp, nil +} + +func (obsClient ObsClient) doHTTP(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (resp *http.Response, respError error) { + + bucketName = strings.TrimSpace(bucketName) + + method = strings.ToUpper(method) + + var redirectURL string + var requestURL string + maxRetryCount := obsClient.conf.maxRetryCount + maxRedirectCount := obsClient.conf.maxRedirectCount + + _data, _err := prepareData(headers, data) + if _err != nil { + return nil, _err + } + + var lastRequest *http.Request + redirectFlag := false + for i, redirectCount := 0, 0; i <= maxRetryCount; i++ { + req, err := obsClient.getRequest(redirectURL, requestURL, redirectFlag, _data, + method, bucketName, objectKey, params, headers) + if err != nil { + return nil, err + } + + logHeaders(headers, obsClient.conf.signature) + + lastRequest = prepareReq(headers, req, lastRequest, obsClient.conf.userAgent) + + start := GetCurrentTimestamp() + resp, err = obsClient.httpClient.Do(req) + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + + var msg interface{} + if err != nil { + msg = err + respError = err + resp = nil + if !repeatable { + break + } + } else { + doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) + if resp.StatusCode < 300 { + respError = nil + break + } else if canNotRetry(repeatable, resp.StatusCode) { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + break + } else if resp.StatusCode >= 300 && resp.StatusCode < 400 { + location := resp.Header.Get(HEADER_LOCATION_CAMEL) + if isRedirectErr(location, redirectCount, maxRedirectCount) { + redirectURL = location + doLog(LEVEL_WARN, "Redirect request to %s", redirectURL) + msg = resp.Status + maxRetryCount++ + redirectCount++ + redirectFlag = setRedirectFlag(resp.StatusCode, method) + } else { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + break + } + } else { + msg = resp.Status + } + } + if i != maxRetryCount { + _data, resp, err = prepareRetry(resp, headers, _data, msg) + if err != nil { + return nil, err + } + if r, ok := _data.(*fileReaderWrapper); ok { + if _fd, _ok := r.reader.(*os.File); _ok { + defer func() { + errMsg := _fd.Close() + checkAndLogErr(errMsg, LEVEL_WARN, "Failed to close with reason: %v", errMsg) + }() + } + } + time.Sleep(time.Duration(float64(i+2) * rand.Float64() * float64(time.Second))) + } else { + doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) + if resp != nil { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + } + } + } + return +} + +type connDelegate struct { + conn net.Conn + socketTimeout time.Duration + finalTimeout time.Duration +} + +func getConnDelegate(conn net.Conn, socketTimeout int, finalTimeout int) *connDelegate { + return &connDelegate{ + conn: conn, + socketTimeout: time.Second * time.Duration(socketTimeout), + finalTimeout: time.Second * time.Duration(finalTimeout), + } +} + +func (delegate *connDelegate) Read(b []byte) (n int, err error) { + setReadDeadlineErr := delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + + n, err = delegate.conn.Read(b) + setReadDeadlineErr = delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout)) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + return n, err +} + +func (delegate *connDelegate) Write(b []byte) (n int, err error) { + setWriteDeadlineErr := delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + + n, err = delegate.conn.Write(b) + finalTimeout := time.Now().Add(delegate.finalTimeout) + setWriteDeadlineErr = delegate.SetWriteDeadline(finalTimeout) + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + setReadDeadlineErr := delegate.SetReadDeadline(finalTimeout) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + return n, err +} + +func (delegate *connDelegate) Close() error { + return delegate.conn.Close() +} + +func (delegate *connDelegate) LocalAddr() net.Addr { + return delegate.conn.LocalAddr() +} + +func (delegate *connDelegate) RemoteAddr() net.Addr { + return delegate.conn.RemoteAddr() +} + +func (delegate *connDelegate) SetDeadline(t time.Time) error { + return delegate.conn.SetDeadline(t) +} + +func (delegate *connDelegate) SetReadDeadline(t time.Time) error { + return delegate.conn.SetReadDeadline(t) +} + +func (delegate *connDelegate) SetWriteDeadline(t time.Time) error { + return delegate.conn.SetWriteDeadline(t) +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/log.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/log.go new file mode 100644 index 0000000000..781ac2768e --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/log.go @@ -0,0 +1,330 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" + "sync" +) + +// Level defines the level of the log +type Level int + +const ( + LEVEL_OFF Level = 500 + LEVEL_ERROR Level = 400 + LEVEL_WARN Level = 300 + LEVEL_INFO Level = 200 + LEVEL_DEBUG Level = 100 +) + +var logLevelMap = map[Level]string{ + LEVEL_OFF: "[OFF]: ", + LEVEL_ERROR: "[ERROR]: ", + LEVEL_WARN: "[WARN]: ", + LEVEL_INFO: "[INFO]: ", + LEVEL_DEBUG: "[DEBUG]: ", +} + +type logConfType struct { + level Level + logToConsole bool + logFullPath string + maxLogSize int64 + backups int +} + +func getDefaultLogConf() logConfType { + return logConfType{ + level: LEVEL_WARN, + logToConsole: false, + logFullPath: "", + maxLogSize: 1024 * 1024 * 30, //30MB + backups: 10, + } +} + +var logConf logConfType + +type loggerWrapper struct { + fullPath string + fd *os.File + ch chan string + wg sync.WaitGroup + queue []string + logger *log.Logger + index int + cacheCount int + closed bool +} + +func (lw *loggerWrapper) doInit() { + lw.queue = make([]string, 0, lw.cacheCount) + lw.logger = log.New(lw.fd, "", 0) + lw.ch = make(chan string, lw.cacheCount) + lw.wg.Add(1) + go lw.doWrite() +} + +func (lw *loggerWrapper) rotate() { + stat, err := lw.fd.Stat() + if err != nil { + _err := lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + panic(err) + } + if stat.Size() >= logConf.maxLogSize { + _err := lw.fd.Sync() + if _err != nil { + panic(err) + } + _err = lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + if lw.index > logConf.backups { + lw.index = 1 + } + _err = os.Rename(lw.fullPath, lw.fullPath+"."+IntToString(lw.index)) + if _err != nil { + panic(err) + } + lw.index++ + + fd, err := os.OpenFile(lw.fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + panic(err) + } + lw.fd = fd + lw.logger.SetOutput(lw.fd) + } +} + +func (lw *loggerWrapper) doFlush() { + lw.rotate() + for _, m := range lw.queue { + lw.logger.Println(m) + } + err := lw.fd.Sync() + if err != nil { + panic(err) + } +} + +func (lw *loggerWrapper) doClose() { + lw.closed = true + close(lw.ch) + lw.wg.Wait() +} + +func (lw *loggerWrapper) doWrite() { + defer lw.wg.Done() + for { + msg, ok := <-lw.ch + if !ok { + lw.doFlush() + _err := lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + break + } + if len(lw.queue) >= lw.cacheCount { + lw.doFlush() + lw.queue = make([]string, 0, lw.cacheCount) + } + lw.queue = append(lw.queue, msg) + } + +} + +func (lw *loggerWrapper) Printf(format string, v ...interface{}) { + if !lw.closed { + msg := fmt.Sprintf(format, v...) + lw.ch <- msg + } +} + +var consoleLogger *log.Logger +var fileLogger *loggerWrapper +var lock = new(sync.RWMutex) + +func isDebugLogEnabled() bool { + return logConf.level <= LEVEL_DEBUG +} + +func isErrorLogEnabled() bool { + return logConf.level <= LEVEL_ERROR +} + +func isWarnLogEnabled() bool { + return logConf.level <= LEVEL_WARN +} + +func isInfoLogEnabled() bool { + return logConf.level <= LEVEL_INFO +} + +func reset() { + if fileLogger != nil { + fileLogger.doClose() + fileLogger = nil + } + consoleLogger = nil + logConf = getDefaultLogConf() +} + +// InitLog enable logging function with default cacheCnt +func InitLog(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool) error { + return InitLogWithCacheCnt(logFullPath, maxLogSize, backups, level, logToConsole, 50) +} + +// InitLogWithCacheCnt enable logging function +func InitLogWithCacheCnt(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool, cacheCnt int) error { + lock.Lock() + defer lock.Unlock() + if cacheCnt <= 0 { + cacheCnt = 50 + } + reset() + if fullPath := strings.TrimSpace(logFullPath); fullPath != "" { + _fullPath, err := filepath.Abs(fullPath) + if err != nil { + return err + } + + if !strings.HasSuffix(_fullPath, ".log") { + _fullPath += ".log" + } + + stat, fd, err := initLogFile(_fullPath) + if err != nil { + return err + } + + prefix := stat.Name() + "." + index := 1 + var timeIndex int64 = 0 + walkFunc := func(path string, info os.FileInfo, err error) error { + if err == nil { + if name := info.Name(); strings.HasPrefix(name, prefix) { + if i := StringToInt(name[len(prefix):], 0); i >= index && info.ModTime().Unix() >= timeIndex { + timeIndex = info.ModTime().Unix() + index = i + 1 + } + } + } + return err + } + + if err = filepath.Walk(filepath.Dir(_fullPath), walkFunc); err != nil { + _err := fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + return err + } + + fileLogger = &loggerWrapper{fullPath: _fullPath, fd: fd, index: index, cacheCount: cacheCnt, closed: false} + fileLogger.doInit() + } + if maxLogSize > 0 { + logConf.maxLogSize = maxLogSize + } + if backups > 0 { + logConf.backups = backups + } + logConf.level = level + if logToConsole { + consoleLogger = log.New(os.Stdout, "", log.LstdFlags) + } + return nil +} + +func initLogFile(_fullPath string) (os.FileInfo, *os.File, error) { + stat, err := os.Stat(_fullPath) + if err == nil && stat.IsDir() { + return nil, nil, fmt.Errorf("logFullPath:[%s] is a directory", _fullPath) + } else if err = os.MkdirAll(filepath.Dir(_fullPath), os.ModePerm); err != nil { + return nil, nil, err + } + + fd, err := os.OpenFile(_fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return nil, nil, err + } + + if stat == nil { + stat, err = os.Stat(_fullPath) + if err != nil { + _err := fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } + return nil, nil, err + } + } + return stat, fd, nil +} + +// CloseLog disable logging and synchronize cache data to log files +func CloseLog() { + if logEnabled() { + lock.Lock() + defer lock.Unlock() + reset() + } +} + +func logEnabled() bool { + return consoleLogger != nil || fileLogger != nil +} + +// DoLog writes log messages to the logger +func DoLog(level Level, format string, v ...interface{}) { + doLog(level, format, v...) +} + +func doLog(level Level, format string, v ...interface{}) { + if logEnabled() && logConf.level <= level { + msg := fmt.Sprintf(format, v...) + if _, file, line, ok := runtime.Caller(1); ok { + index := strings.LastIndex(file, "/") + if index >= 0 { + file = file[index+1:] + } + msg = fmt.Sprintf("%s:%d|%s", file, line, msg) + } + prefix := logLevelMap[level] + if consoleLogger != nil { + consoleLogger.Printf("%s%s", prefix, msg) + } + if fileLogger != nil { + nowDate := FormatUtcNow("2006-01-02T15:04:05Z") + fileLogger.Printf("%s %s%s", nowDate, prefix, msg) + } + } +} + +func checkAndLogErr(err error, level Level, format string, v ...interface{}) { + if err != nil { + doLog(level, format, v...) + } +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_base.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_base.go new file mode 100644 index 0000000000..c210a338ce --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_base.go @@ -0,0 +1,363 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +package obs + +import ( + "encoding/xml" + "time" +) + +// Bucket defines bucket properties +type Bucket struct { + XMLName xml.Name `xml:"Bucket"` + Name string `xml:"Name"` + CreationDate time.Time `xml:"CreationDate"` + Location string `xml:"Location"` + BucketType string `xml:"BucketType,omitempty"` +} + +// Owner defines owner properties +type Owner struct { + XMLName xml.Name `xml:"Owner"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName,omitempty"` +} + +// Initiator defines initiator properties +type Initiator struct { + XMLName xml.Name `xml:"Initiator"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName,omitempty"` +} + +type bucketLocationObs struct { + XMLName xml.Name `xml:"Location"` + Location string `xml:",chardata"` +} + +// BucketLocation defines bucket location configuration +type BucketLocation struct { + XMLName xml.Name `xml:"CreateBucketConfiguration"` + Location string `xml:"LocationConstraint,omitempty"` +} + +// BucketStoragePolicy defines the bucket storage class +type BucketStoragePolicy struct { + XMLName xml.Name `xml:"StoragePolicy"` + StorageClass StorageClassType `xml:"DefaultStorageClass"` +} + +type bucketStoragePolicyObs struct { + XMLName xml.Name `xml:"StorageClass"` + StorageClass string `xml:",chardata"` +} + +// Content defines the object content properties +type Content struct { + XMLName xml.Name `xml:"Contents"` + Owner Owner `xml:"Owner"` + ETag string `xml:"ETag"` + Key string `xml:"Key"` + LastModified time.Time `xml:"LastModified"` + Size int64 `xml:"Size"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// Version defines the properties of versioning objects +type Version struct { + DeleteMarker + XMLName xml.Name `xml:"Version"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` +} + +// DeleteMarker defines the properties of versioning delete markers +type DeleteMarker struct { + XMLName xml.Name `xml:"DeleteMarker"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified time.Time `xml:"LastModified"` + Owner Owner `xml:"Owner"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// Upload defines multipart upload properties +type Upload struct { + XMLName xml.Name `xml:"Upload"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + Initiated time.Time `xml:"Initiated"` + StorageClass StorageClassType `xml:"StorageClass"` + Owner Owner `xml:"Owner"` + Initiator Initiator `xml:"Initiator"` +} + +// BucketQuota defines bucket quota configuration +type BucketQuota struct { + XMLName xml.Name `xml:"Quota"` + Quota int64 `xml:"StorageQuota"` +} + +// Grantee defines grantee properties +type Grantee struct { + XMLName xml.Name `xml:"Grantee"` + Type GranteeType `xml:"type,attr"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + URI GroupUriType `xml:"URI,omitempty"` +} + +type granteeObs struct { + XMLName xml.Name `xml:"Grantee"` + Type GranteeType `xml:"type,attr"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + Canned string `xml:"Canned,omitempty"` +} + +// Grant defines grant properties +type Grant struct { + XMLName xml.Name `xml:"Grant"` + Grantee Grantee `xml:"Grantee"` + Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` +} + +type grantObs struct { + XMLName xml.Name `xml:"Grant"` + Grantee granteeObs `xml:"Grantee"` + Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` +} + +// AccessControlPolicy defines access control policy properties +type AccessControlPolicy struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []Grant `xml:"AccessControlList>Grant"` + Delivered string `xml:"Delivered,omitempty"` +} + +type accessControlPolicyObs struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []grantObs `xml:"AccessControlList>Grant"` +} + +// CorsRule defines the CORS rules +type CorsRule struct { + XMLName xml.Name `xml:"CORSRule"` + ID string `xml:"ID,omitempty"` + AllowedOrigin []string `xml:"AllowedOrigin"` + AllowedMethod []string `xml:"AllowedMethod"` + AllowedHeader []string `xml:"AllowedHeader,omitempty"` + MaxAgeSeconds int `xml:"MaxAgeSeconds"` + ExposeHeader []string `xml:"ExposeHeader,omitempty"` +} + +// BucketCors defines the bucket CORS configuration +type BucketCors struct { + XMLName xml.Name `xml:"CORSConfiguration"` + CorsRules []CorsRule `xml:"CORSRule"` +} + +// BucketVersioningConfiguration defines the versioning configuration +type BucketVersioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status VersioningStatusType `xml:"Status"` +} + +// IndexDocument defines the default page configuration +type IndexDocument struct { + Suffix string `xml:"Suffix"` +} + +// ErrorDocument defines the error page configuration +type ErrorDocument struct { + Key string `xml:"Key,omitempty"` +} + +// Condition defines condition in RoutingRule +type Condition struct { + XMLName xml.Name `xml:"Condition"` + KeyPrefixEquals string `xml:"KeyPrefixEquals,omitempty"` + HttpErrorCodeReturnedEquals string `xml:"HttpErrorCodeReturnedEquals,omitempty"` +} + +// Redirect defines redirect in RoutingRule +type Redirect struct { + XMLName xml.Name `xml:"Redirect"` + Protocol ProtocolType `xml:"Protocol,omitempty"` + HostName string `xml:"HostName,omitempty"` + ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith,omitempty"` + ReplaceKeyWith string `xml:"ReplaceKeyWith,omitempty"` + HttpRedirectCode string `xml:"HttpRedirectCode,omitempty"` +} + +// RoutingRule defines routing rules +type RoutingRule struct { + XMLName xml.Name `xml:"RoutingRule"` + Condition Condition `xml:"Condition,omitempty"` + Redirect Redirect `xml:"Redirect"` +} + +// RedirectAllRequestsTo defines redirect in BucketWebsiteConfiguration +type RedirectAllRequestsTo struct { + XMLName xml.Name `xml:"RedirectAllRequestsTo"` + Protocol ProtocolType `xml:"Protocol,omitempty"` + HostName string `xml:"HostName"` +} + +// BucketWebsiteConfiguration defines the bucket website configuration +type BucketWebsiteConfiguration struct { + XMLName xml.Name `xml:"WebsiteConfiguration"` + RedirectAllRequestsTo RedirectAllRequestsTo `xml:"RedirectAllRequestsTo,omitempty"` + IndexDocument IndexDocument `xml:"IndexDocument,omitempty"` + ErrorDocument ErrorDocument `xml:"ErrorDocument,omitempty"` + RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` +} + +// BucketLoggingStatus defines the bucket logging configuration +type BucketLoggingStatus struct { + XMLName xml.Name `xml:"BucketLoggingStatus"` + Agency string `xml:"Agency,omitempty"` + TargetBucket string `xml:"LoggingEnabled>TargetBucket,omitempty"` + TargetPrefix string `xml:"LoggingEnabled>TargetPrefix,omitempty"` + TargetGrants []Grant `xml:"LoggingEnabled>TargetGrants>Grant,omitempty"` +} + +// Transition defines transition property in LifecycleRule +type Transition struct { + XMLName xml.Name `xml:"Transition"` + Date time.Time `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// Expiration defines expiration property in LifecycleRule +type Expiration struct { + XMLName xml.Name `xml:"Expiration"` + Date time.Time `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` +} + +// NoncurrentVersionTransition defines noncurrentVersion transition property in LifecycleRule +type NoncurrentVersionTransition struct { + XMLName xml.Name `xml:"NoncurrentVersionTransition"` + NoncurrentDays int `xml:"NoncurrentDays"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +// NoncurrentVersionExpiration defines noncurrentVersion expiration property in LifecycleRule +type NoncurrentVersionExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration"` + NoncurrentDays int `xml:"NoncurrentDays"` +} + +// LifecycleRule defines lifecycle rule +type LifecycleRule struct { + ID string `xml:"ID,omitempty"` + Prefix string `xml:"Prefix"` + Status RuleStatusType `xml:"Status"` + Transitions []Transition `xml:"Transition,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty"` + NoncurrentVersionTransitions []NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"` + NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"` +} + +// BucketEncryptionConfiguration defines the bucket encryption configuration +type BucketEncryptionConfiguration struct { + XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` + SSEAlgorithm string `xml:"Rule>ApplyServerSideEncryptionByDefault>SSEAlgorithm"` + KMSMasterKeyID string `xml:"Rule>ApplyServerSideEncryptionByDefault>KMSMasterKeyID,omitempty"` + ProjectID string `xml:"Rule>ApplyServerSideEncryptionByDefault>ProjectID,omitempty"` +} + +// Tag defines tag property in BucketTagging +type Tag struct { + XMLName xml.Name `xml:"Tag"` + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +// BucketTagging defines the bucket tag configuration +type BucketTagging struct { + XMLName xml.Name `xml:"Tagging"` + Tags []Tag `xml:"TagSet>Tag"` +} + +// FilterRule defines filter rule in TopicConfiguration +type FilterRule struct { + XMLName xml.Name `xml:"FilterRule"` + Name string `xml:"Name,omitempty"` + Value string `xml:"Value,omitempty"` +} + +// TopicConfiguration defines the topic configuration +type TopicConfiguration struct { + XMLName xml.Name `xml:"TopicConfiguration"` + ID string `xml:"Id,omitempty"` + Topic string `xml:"Topic"` + Events []EventType `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>Object>FilterRule"` +} + +// BucketNotification defines the bucket notification configuration +type BucketNotification struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + TopicConfigurations []TopicConfiguration `xml:"TopicConfiguration"` +} + +type topicConfigurationS3 struct { + XMLName xml.Name `xml:"TopicConfiguration"` + ID string `xml:"Id,omitempty"` + Topic string `xml:"Topic"` + Events []string `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>S3Key>FilterRule"` +} + +type bucketNotificationS3 struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + TopicConfigurations []topicConfigurationS3 `xml:"TopicConfiguration"` +} + +// ObjectToDelete defines the object property in DeleteObjectsInput +type ObjectToDelete struct { + XMLName xml.Name `xml:"Object"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` +} + +// Deleted defines the deleted property in DeleteObjectsOutput +type Deleted struct { + XMLName xml.Name `xml:"Deleted"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + DeleteMarker bool `xml:"DeleteMarker"` + DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId"` +} + +// Part defines the part properties +type Part struct { + XMLName xml.Name `xml:"Part"` + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified,omitempty"` + Size int64 `xml:"Size,omitempty"` +} + +// BucketPayer defines the request payment configuration +type BucketPayer struct { + XMLName xml.Name `xml:"RequestPaymentConfiguration"` + Payer PayerType `xml:"Payer"` +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_bucket.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_bucket.go new file mode 100644 index 0000000000..a0ba7f99b8 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_bucket.go @@ -0,0 +1,356 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +package obs + +import ( + "encoding/xml" +) + +// ListBucketsInput is the input parameter of ListBuckets function +type ListBucketsInput struct { + QueryLocation bool + BucketType BucketType +} + +// ListBucketsOutput is the result of ListBuckets function +type ListBucketsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Owner Owner `xml:"Owner"` + Buckets []Bucket `xml:"Buckets>Bucket"` +} + +// CreateBucketInput is the input parameter of CreateBucket function +type CreateBucketInput struct { + BucketLocation + Bucket string `xml:"-"` + ACL AclType `xml:"-"` + StorageClass StorageClassType `xml:"-"` + GrantReadId string `xml:"-"` + GrantWriteId string `xml:"-"` + GrantReadAcpId string `xml:"-"` + GrantWriteAcpId string `xml:"-"` + GrantFullControlId string `xml:"-"` + GrantReadDeliveredId string `xml:"-"` + GrantFullControlDeliveredId string `xml:"-"` + Epid string `xml:"-"` + AvailableZone string `xml:"-"` + IsFSFileInterface bool `xml:"-"` +} + +// SetBucketStoragePolicyInput is the input parameter of SetBucketStoragePolicy function +type SetBucketStoragePolicyInput struct { + Bucket string `xml:"-"` + BucketStoragePolicy +} + +type getBucketStoragePolicyOutputS3 struct { + BaseModel + BucketStoragePolicy +} + +// GetBucketStoragePolicyOutput is the result of GetBucketStoragePolicy function +type GetBucketStoragePolicyOutput struct { + BaseModel + StorageClass string +} + +type getBucketStoragePolicyOutputObs struct { + BaseModel + bucketStoragePolicyObs +} + +// SetBucketQuotaInput is the input parameter of SetBucketQuota function +type SetBucketQuotaInput struct { + Bucket string `xml:"-"` + BucketQuota +} + +// GetBucketQuotaOutput is the result of GetBucketQuota function +type GetBucketQuotaOutput struct { + BaseModel + BucketQuota +} + +// GetBucketStorageInfoOutput is the result of GetBucketStorageInfo function +type GetBucketStorageInfoOutput struct { + BaseModel + XMLName xml.Name `xml:"GetBucketStorageInfoResult"` + Size int64 `xml:"Size"` + ObjectNumber int `xml:"ObjectNumber"` +} + +type getBucketLocationOutputS3 struct { + BaseModel + BucketLocation +} +type getBucketLocationOutputObs struct { + BaseModel + bucketLocationObs +} + +// GetBucketLocationOutput is the result of GetBucketLocation function +type GetBucketLocationOutput struct { + BaseModel + Location string `xml:"-"` +} + +// GetBucketAclOutput is the result of GetBucketAcl function +type GetBucketAclOutput struct { + BaseModel + AccessControlPolicy +} + +type getBucketACLOutputObs struct { + BaseModel + accessControlPolicyObs +} + +// SetBucketAclInput is the input parameter of SetBucketAcl function +type SetBucketAclInput struct { + Bucket string `xml:"-"` + ACL AclType `xml:"-"` + AccessControlPolicy +} + +// SetBucketPolicyInput is the input parameter of SetBucketPolicy function +type SetBucketPolicyInput struct { + Bucket string + Policy string +} + +// GetBucketPolicyOutput is the result of GetBucketPolicy function +type GetBucketPolicyOutput struct { + BaseModel + Policy string `json:"body"` +} + +// SetBucketCorsInput is the input parameter of SetBucketCors function +type SetBucketCorsInput struct { + Bucket string `xml:"-"` + BucketCors +} + +// GetBucketCorsOutput is the result of GetBucketCors function +type GetBucketCorsOutput struct { + BaseModel + BucketCors +} + +// SetBucketVersioningInput is the input parameter of SetBucketVersioning function +type SetBucketVersioningInput struct { + Bucket string `xml:"-"` + BucketVersioningConfiguration +} + +// GetBucketVersioningOutput is the result of GetBucketVersioning function +type GetBucketVersioningOutput struct { + BaseModel + BucketVersioningConfiguration +} + +// SetBucketWebsiteConfigurationInput is the input parameter of SetBucketWebsiteConfiguration function +type SetBucketWebsiteConfigurationInput struct { + Bucket string `xml:"-"` + BucketWebsiteConfiguration +} + +// GetBucketWebsiteConfigurationOutput is the result of GetBucketWebsiteConfiguration function +type GetBucketWebsiteConfigurationOutput struct { + BaseModel + BucketWebsiteConfiguration +} + +// GetBucketMetadataInput is the input parameter of GetBucketMetadata function +type GetBucketMetadataInput struct { + Bucket string + Origin string + RequestHeader string +} + +// SetObjectMetadataInput is the input parameter of SetObjectMetadata function +type SetObjectMetadataInput struct { + Bucket string + Key string + VersionId string + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + +//SetObjectMetadataOutput is the result of SetObjectMetadata function +type SetObjectMetadataOutput struct { + BaseModel + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + +// GetBucketMetadataOutput is the result of GetBucketMetadata function +type GetBucketMetadataOutput struct { + BaseModel + StorageClass StorageClassType + Location string + Version string + AllowOrigin string + AllowMethod string + AllowHeader string + MaxAgeSeconds int + ExposeHeader string + Epid string + AZRedundancy string + FSStatus FSStatusType +} + +// SetBucketLoggingConfigurationInput is the input parameter of SetBucketLoggingConfiguration function +type SetBucketLoggingConfigurationInput struct { + Bucket string `xml:"-"` + BucketLoggingStatus +} + +// GetBucketLoggingConfigurationOutput is the result of GetBucketLoggingConfiguration function +type GetBucketLoggingConfigurationOutput struct { + BaseModel + BucketLoggingStatus +} + +// BucketLifecyleConfiguration defines the bucket lifecycle configuration +type BucketLifecyleConfiguration struct { + XMLName xml.Name `xml:"LifecycleConfiguration"` + LifecycleRules []LifecycleRule `xml:"Rule"` +} + +// SetBucketLifecycleConfigurationInput is the input parameter of SetBucketLifecycleConfiguration function +type SetBucketLifecycleConfigurationInput struct { + Bucket string `xml:"-"` + BucketLifecyleConfiguration +} + +// GetBucketLifecycleConfigurationOutput is the result of GetBucketLifecycleConfiguration function +type GetBucketLifecycleConfigurationOutput struct { + BaseModel + BucketLifecyleConfiguration +} + +// SetBucketEncryptionInput is the input parameter of SetBucketEncryption function +type SetBucketEncryptionInput struct { + Bucket string `xml:"-"` + BucketEncryptionConfiguration +} + +// GetBucketEncryptionOutput is the result of GetBucketEncryption function +type GetBucketEncryptionOutput struct { + BaseModel + BucketEncryptionConfiguration +} + +// SetBucketTaggingInput is the input parameter of SetBucketTagging function +type SetBucketTaggingInput struct { + Bucket string `xml:"-"` + BucketTagging +} + +// GetBucketTaggingOutput is the result of GetBucketTagging function +type GetBucketTaggingOutput struct { + BaseModel + BucketTagging +} + +// SetBucketNotificationInput is the input parameter of SetBucketNotification function +type SetBucketNotificationInput struct { + Bucket string `xml:"-"` + BucketNotification +} + +type getBucketNotificationOutputS3 struct { + BaseModel + bucketNotificationS3 +} + +// GetBucketNotificationOutput is the result of GetBucketNotification function +type GetBucketNotificationOutput struct { + BaseModel + BucketNotification +} + +// SetBucketFetchPolicyInput is the input parameter of SetBucketFetchPolicy function +type SetBucketFetchPolicyInput struct { + Bucket string + Status FetchPolicyStatusType `json:"status"` + Agency string `json:"agency"` +} + +// GetBucketFetchPolicyInput is the input parameter of GetBucketFetchPolicy function +type GetBucketFetchPolicyInput struct { + Bucket string +} + +// GetBucketFetchPolicyOutput is the result of GetBucketFetchPolicy function +type GetBucketFetchPolicyOutput struct { + BaseModel + FetchResponse `json:"fetch"` +} + +// DeleteBucketFetchPolicyInput is the input parameter of DeleteBucketFetchPolicy function +type DeleteBucketFetchPolicyInput struct { + Bucket string +} + +// SetBucketFetchJobInput is the input parameter of SetBucketFetchJob function +type SetBucketFetchJobInput struct { + Bucket string `json:"bucket"` + URL string `json:"url"` + Host string `json:"host,omitempty"` + Key string `json:"key,omitempty"` + Md5 string `json:"md5,omitempty"` + CallBackURL string `json:"callbackurl,omitempty"` + CallBackBody string `json:"callbackbody,omitempty"` + CallBackBodyType string `json:"callbackbodytype,omitempty"` + CallBackHost string `json:"callbackhost,omitempty"` + FileType string `json:"file_type,omitempty"` + IgnoreSameKey bool `json:"ignore_same_key,omitempty"` + ObjectHeaders map[string]string `json:"objectheaders,omitempty"` + Etag string `json:"etag,omitempty"` + TrustName string `json:"trustname,omitempty"` +} + +// SetBucketFetchJobOutput is the result of SetBucketFetchJob function +type SetBucketFetchJobOutput struct { + BaseModel + SetBucketFetchJobResponse +} + +// GetBucketFetchJobInput is the input parameter of GetBucketFetchJob function +type GetBucketFetchJobInput struct { + Bucket string + JobID string +} + +// GetBucketFetchJobOutput is the result of GetBucketFetchJob function +type GetBucketFetchJobOutput struct { + BaseModel + GetBucketFetchJobResponse +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_header.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_header.go new file mode 100644 index 0000000000..c76d9f0268 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_header.go @@ -0,0 +1,32 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +package obs + +// ISseHeader defines the sse encryption header +type ISseHeader interface { + GetEncryption() string + GetKey() string +} + +// SseKmsHeader defines the SseKms header +type SseKmsHeader struct { + Encryption string + Key string + isObs bool +} + +// SseCHeader defines the SseC header +type SseCHeader struct { + Encryption string + Key string + KeyMD5 string +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_object.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_object.go new file mode 100644 index 0000000000..b599e89af6 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_object.go @@ -0,0 +1,345 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +package obs + +import ( + "encoding/xml" + "io" + "time" +) + +// ListObjsInput defines parameters for listing objects +type ListObjsInput struct { + Prefix string + MaxKeys int + Delimiter string + Origin string + RequestHeader string + EncodingType string +} + +// ListObjectsInput is the input parameter of ListObjects function +type ListObjectsInput struct { + ListObjsInput + Bucket string + Marker string +} + +// ListObjectsOutput is the result of ListObjects function +type ListObjectsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListBucketResult"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + Marker string `xml:"Marker"` + NextMarker string `xml:"NextMarker"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Contents []Content `xml:"Contents"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + Location string `xml:"-"` + EncodingType string `xml:"EncodingType,omitempty"` +} + +// ListVersionsInput is the input parameter of ListVersions function +type ListVersionsInput struct { + ListObjsInput + Bucket string + KeyMarker string + VersionIdMarker string +} + +// ListVersionsOutput is the result of ListVersions function +type ListVersionsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListVersionsResult"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + KeyMarker string `xml:"KeyMarker"` + NextKeyMarker string `xml:"NextKeyMarker"` + VersionIdMarker string `xml:"VersionIdMarker"` + NextVersionIdMarker string `xml:"NextVersionIdMarker"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Versions []Version `xml:"Version"` + DeleteMarkers []DeleteMarker `xml:"DeleteMarker"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + Location string `xml:"-"` + EncodingType string `xml:"EncodingType,omitempty"` +} + +// DeleteObjectInput is the input parameter of DeleteObject function +type DeleteObjectInput struct { + Bucket string + Key string + VersionId string +} + +// DeleteObjectOutput is the result of DeleteObject function +type DeleteObjectOutput struct { + BaseModel + VersionId string + DeleteMarker bool +} + +// DeleteObjectsInput is the input parameter of DeleteObjects function +type DeleteObjectsInput struct { + Bucket string `xml:"-"` + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet,omitempty"` + Objects []ObjectToDelete `xml:"Object"` + EncodingType string `xml:"EncodingType"` +} + +// DeleteObjectsOutput is the result of DeleteObjects function +type DeleteObjectsOutput struct { + BaseModel + XMLName xml.Name `xml:"DeleteResult"` + Deleteds []Deleted `xml:"Deleted"` + Errors []Error `xml:"Error"` + EncodingType string `xml:"EncodingType,omitempty"` +} + +// SetObjectAclInput is the input parameter of SetObjectAcl function +type SetObjectAclInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + VersionId string `xml:"-"` + ACL AclType `xml:"-"` + AccessControlPolicy +} + +// GetObjectAclInput is the input parameter of GetObjectAcl function +type GetObjectAclInput struct { + Bucket string + Key string + VersionId string +} + +// GetObjectAclOutput is the result of GetObjectAcl function +type GetObjectAclOutput struct { + BaseModel + VersionId string + AccessControlPolicy +} + +// RestoreObjectInput is the input parameter of RestoreObject function +type RestoreObjectInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + VersionId string `xml:"-"` + XMLName xml.Name `xml:"RestoreRequest"` + Days int `xml:"Days"` + Tier RestoreTierType `xml:"GlacierJobParameters>Tier,omitempty"` +} + +// GetObjectMetadataInput is the input parameter of GetObjectMetadata function +type GetObjectMetadataInput struct { + Bucket string + Key string + VersionId string + Origin string + RequestHeader string + SseHeader ISseHeader +} + +// GetObjectMetadataOutput is the result of GetObjectMetadata function +type GetObjectMetadataOutput struct { + BaseModel + VersionId string + WebsiteRedirectLocation string + Expiration string + Restore string + ObjectType string + NextAppendPosition string + StorageClass StorageClassType + ContentLength int64 + ContentType string + ETag string + AllowOrigin string + AllowHeader string + AllowMethod string + ExposeHeader string + MaxAgeSeconds int + LastModified time.Time + SseHeader ISseHeader + Metadata map[string]string +} + +// GetObjectInput is the input parameter of GetObject function +type GetObjectInput struct { + GetObjectMetadataInput + IfMatch string + IfNoneMatch string + IfUnmodifiedSince time.Time + IfModifiedSince time.Time + RangeStart int64 + RangeEnd int64 + ImageProcess string + ResponseCacheControl string + ResponseContentDisposition string + ResponseContentEncoding string + ResponseContentLanguage string + ResponseContentType string + ResponseExpires string +} + +// GetObjectOutput is the result of GetObject function +type GetObjectOutput struct { + GetObjectMetadataOutput + DeleteMarker bool + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + Expires string + Body io.ReadCloser +} + +// ObjectOperationInput defines the object operation properties +type ObjectOperationInput struct { + Bucket string + Key string + ACL AclType + GrantReadId string + GrantReadAcpId string + GrantWriteAcpId string + GrantFullControlId string + StorageClass StorageClassType + WebsiteRedirectLocation string + Expires int64 + SseHeader ISseHeader + Metadata map[string]string +} + +// PutObjectBasicInput defines the basic object operation properties +type PutObjectBasicInput struct { + ObjectOperationInput + ContentType string + ContentMD5 string + ContentLength int64 +} + +// PutObjectInput is the input parameter of PutObject function +type PutObjectInput struct { + PutObjectBasicInput + Body io.Reader +} + +// PutFileInput is the input parameter of PutFile function +type PutFileInput struct { + PutObjectBasicInput + SourceFile string +} + +// PutObjectOutput is the result of PutObject function +type PutObjectOutput struct { + BaseModel + VersionId string + SseHeader ISseHeader + StorageClass StorageClassType + ETag string +} + +// CopyObjectInput is the input parameter of CopyObject function +type CopyObjectInput struct { + ObjectOperationInput + CopySourceBucket string + CopySourceKey string + CopySourceVersionId string + CopySourceIfMatch string + CopySourceIfNoneMatch string + CopySourceIfUnmodifiedSince time.Time + CopySourceIfModifiedSince time.Time + SourceSseHeader ISseHeader + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + MetadataDirective MetadataDirectiveType + SuccessActionRedirect string +} + +// CopyObjectOutput is the result of CopyObject function +type CopyObjectOutput struct { + BaseModel + CopySourceVersionId string `xml:"-"` + VersionId string `xml:"-"` + SseHeader ISseHeader `xml:"-"` + XMLName xml.Name `xml:"CopyObjectResult"` + LastModified time.Time `xml:"LastModified"` + ETag string `xml:"ETag"` +} + +// UploadFileInput is the input parameter of UploadFile function +type UploadFileInput struct { + ObjectOperationInput + ContentType string + UploadFile string + PartSize int64 + TaskNum int + EnableCheckpoint bool + CheckpointFile string + EncodingType string +} + +// DownloadFileInput is the input parameter of DownloadFile function +type DownloadFileInput struct { + GetObjectMetadataInput + IfMatch string + IfNoneMatch string + IfModifiedSince time.Time + IfUnmodifiedSince time.Time + DownloadFile string + PartSize int64 + TaskNum int + EnableCheckpoint bool + CheckpointFile string +} + +type AppendObjectInput struct { + PutObjectBasicInput + Body io.Reader + Position int64 +} + +type AppendObjectOutput struct { + BaseModel + VersionId string + SseHeader ISseHeader + NextAppendPosition int64 + ETag string +} + +type ModifyObjectInput struct { + Bucket string + Key string + Position int64 + Body io.Reader + ContentLength int64 +} + +type ModifyObjectOutput struct { + BaseModel + ETag string +} + +// HeadObjectInput is the input parameter of HeadObject function +type HeadObjectInput struct { + Bucket string + Key string + VersionId string +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_other.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_other.go new file mode 100644 index 0000000000..8ede31d986 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_other.go @@ -0,0 +1,63 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +package obs + +import ( + "net/http" +) + +// CreateSignedUrlInput is the input parameter of CreateSignedUrl function +type CreateSignedUrlInput struct { + Method HttpMethodType + Bucket string + Key string + SubResource SubResourceType + Expires int + Headers map[string]string + QueryParams map[string]string +} + +// CreateSignedUrlOutput is the result of CreateSignedUrl function +type CreateSignedUrlOutput struct { + SignedUrl string + ActualSignedRequestHeaders http.Header +} + +// CreateBrowserBasedSignatureInput is the input parameter of CreateBrowserBasedSignature function. +type CreateBrowserBasedSignatureInput struct { + Bucket string + Key string + Expires int + FormParams map[string]string +} + +// CreateBrowserBasedSignatureOutput is the result of CreateBrowserBasedSignature function. +type CreateBrowserBasedSignatureOutput struct { + OriginPolicy string + Policy string + Algorithm string + Credential string + Date string + Signature string +} + +// SetBucketRequestPaymentInput is the input parameter of SetBucketRequestPayment function +type SetBucketRequestPaymentInput struct { + Bucket string `xml:"-"` + BucketPayer +} + +// GetBucketRequestPaymentOutput is the result of GetBucketRequestPayment function +type GetBucketRequestPaymentOutput struct { + BaseModel + BucketPayer +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_part.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_part.go new file mode 100644 index 0000000000..a6752b8873 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_part.go @@ -0,0 +1,170 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +package obs + +import ( + "encoding/xml" + "io" + "time" +) + +// ListMultipartUploadsInput is the input parameter of ListMultipartUploads function +type ListMultipartUploadsInput struct { + Bucket string + Prefix string + MaxUploads int + Delimiter string + KeyMarker string + UploadIdMarker string + EncodingType string +} + +// ListMultipartUploadsOutput is the result of ListMultipartUploads function +type ListMultipartUploadsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListMultipartUploadsResult"` + Bucket string `xml:"Bucket"` + KeyMarker string `xml:"KeyMarker"` + NextKeyMarker string `xml:"NextKeyMarker"` + UploadIdMarker string `xml:"UploadIdMarker"` + NextUploadIdMarker string `xml:"NextUploadIdMarker"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + MaxUploads int `xml:"MaxUploads"` + Prefix string `xml:"Prefix"` + Uploads []Upload `xml:"Upload"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + EncodingType string `xml:"EncodingType,omitempty"` +} + +// AbortMultipartUploadInput is the input parameter of AbortMultipartUpload function +type AbortMultipartUploadInput struct { + Bucket string + Key string + UploadId string +} + +// InitiateMultipartUploadInput is the input parameter of InitiateMultipartUpload function +type InitiateMultipartUploadInput struct { + ObjectOperationInput + ContentType string + EncodingType string +} + +// InitiateMultipartUploadOutput is the result of InitiateMultipartUpload function +type InitiateMultipartUploadOutput struct { + BaseModel + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + SseHeader ISseHeader + EncodingType string `xml:"EncodingType,omitempty"` +} + +// UploadPartInput is the input parameter of UploadPart function +type UploadPartInput struct { + Bucket string + Key string + PartNumber int + UploadId string + ContentMD5 string + SseHeader ISseHeader + Body io.Reader + SourceFile string + Offset int64 + PartSize int64 +} + +// UploadPartOutput is the result of UploadPart function +type UploadPartOutput struct { + BaseModel + PartNumber int + ETag string + SseHeader ISseHeader +} + +// CompleteMultipartUploadInput is the input parameter of CompleteMultipartUpload function +type CompleteMultipartUploadInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + UploadId string `xml:"-"` + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []Part `xml:"Part"` + EncodingType string `xml:"-"` +} + +// CompleteMultipartUploadOutput is the result of CompleteMultipartUpload function +type CompleteMultipartUploadOutput struct { + BaseModel + VersionId string `xml:"-"` + SseHeader ISseHeader `xml:"-"` + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` + EncodingType string `xml:"EncodingType,omitempty"` +} + +// ListPartsInput is the input parameter of ListParts function +type ListPartsInput struct { + Bucket string + Key string + UploadId string + MaxParts int + PartNumberMarker int + EncodingType string +} + +// ListPartsOutput is the result of ListParts function +type ListPartsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListPartsResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + PartNumberMarker int `xml:"PartNumberMarker"` + NextPartNumberMarker int `xml:"NextPartNumberMarker"` + MaxParts int `xml:"MaxParts"` + IsTruncated bool `xml:"IsTruncated"` + StorageClass StorageClassType `xml:"StorageClass"` + Initiator Initiator `xml:"Initiator"` + Owner Owner `xml:"Owner"` + Parts []Part `xml:"Part"` + EncodingType string `xml:"EncodingType,omitempty"` +} + +// CopyPartInput is the input parameter of CopyPart function +type CopyPartInput struct { + Bucket string + Key string + UploadId string + PartNumber int + CopySourceBucket string + CopySourceKey string + CopySourceVersionId string + CopySourceRangeStart int64 + CopySourceRangeEnd int64 + SseHeader ISseHeader + SourceSseHeader ISseHeader +} + +// CopyPartOutput is the result of CopyPart function +type CopyPartOutput struct { + BaseModel + XMLName xml.Name `xml:"CopyPartResult"` + PartNumber int `xml:"-"` + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified"` + SseHeader ISseHeader `xml:"-"` +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_response.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_response.go new file mode 100644 index 0000000000..a8db53ee72 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/model_response.go @@ -0,0 +1,67 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +package obs + +import ( + "encoding/xml" +) + +// BaseModel defines base model response from OBS +type BaseModel struct { + StatusCode int `xml:"-"` + RequestId string `xml:"RequestId" json:"request_id"` + ResponseHeaders map[string][]string `xml:"-"` +} + +// Error defines the error property in DeleteObjectsOutput +type Error struct { + XMLName xml.Name `xml:"Error"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +// FetchResponse defines the response fetch policy configuration +type FetchResponse struct { + Status FetchPolicyStatusType `json:"status"` + Agency string `json:"agency"` +} + +// SetBucketFetchJobResponse defines the response SetBucketFetchJob configuration +type SetBucketFetchJobResponse struct { + ID string `json:"id"` + Wait int `json:"Wait"` +} + +// GetBucketFetchJobResponse defines the response fetch job configuration +type GetBucketFetchJobResponse struct { + Err string `json:"err"` + Code string `json:"code"` + Status string `json:"status"` + Job JobResponse `json:"job"` +} + +// JobResponse defines the response job configuration +type JobResponse struct { + Bucket string `json:"bucket"` + URL string `json:"url"` + Host string `json:"host"` + Key string `json:"key"` + Md5 string `json:"md5"` + CallBackURL string `json:"callbackurl"` + CallBackBody string `json:"callbackbody"` + CallBackBodyType string `json:"callbackbodytype"` + CallBackHost string `json:"callbackhost"` + FileType string `json:"file_type"` + IgnoreSameKey bool `json:"ignore_same_key"` +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/pool.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/pool.go new file mode 100644 index 0000000000..1755c96ac5 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/pool.go @@ -0,0 +1,542 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +//nolint:structcheck, unused +package obs + +import ( + "errors" + "fmt" + "runtime" + "sync" + "sync/atomic" + "time" +) + +// Future defines interface with function: Get +type Future interface { + Get() interface{} +} + +// FutureResult for task result +type FutureResult struct { + result interface{} + resultChan chan interface{} + lock sync.Mutex +} + +type panicResult struct { + presult interface{} +} + +func (f *FutureResult) checkPanic() interface{} { + if r, ok := f.result.(panicResult); ok { + panic(r.presult) + } + return f.result +} + +// Get gets the task result +func (f *FutureResult) Get() interface{} { + if f.resultChan == nil { + return f.checkPanic() + } + f.lock.Lock() + defer f.lock.Unlock() + if f.resultChan == nil { + return f.checkPanic() + } + + f.result = <-f.resultChan + close(f.resultChan) + f.resultChan = nil + return f.checkPanic() +} + +// Task defines interface with function: Run +type Task interface { + Run() interface{} +} + +type funcWrapper struct { + f func() interface{} +} + +func (fw *funcWrapper) Run() interface{} { + if fw.f != nil { + return fw.f() + } + return nil +} + +type taskWrapper struct { + t Task + f *FutureResult +} + +func (tw *taskWrapper) Run() interface{} { + if tw.t != nil { + return tw.t.Run() + } + return nil +} + +type signalTask struct { + id string +} + +func (signalTask) Run() interface{} { + return nil +} + +type worker struct { + name string + taskQueue chan Task + wg *sync.WaitGroup + pool *RoutinePool +} + +func runTask(t Task) { + if tw, ok := t.(*taskWrapper); ok { + defer func() { + if r := recover(); r != nil { + tw.f.resultChan <- panicResult{ + presult: r, + } + } + }() + ret := t.Run() + tw.f.resultChan <- ret + } else { + t.Run() + } +} + +func (*worker) runTask(t Task) { + runTask(t) +} + +func (w *worker) start() { + go func() { + defer func() { + if w.wg != nil { + w.wg.Done() + } + }() + for { + task, ok := <-w.taskQueue + if !ok { + break + } + w.pool.AddCurrentWorkingCnt(1) + w.runTask(task) + w.pool.AddCurrentWorkingCnt(-1) + if w.pool.autoTuneWorker(w) { + break + } + } + }() +} + +func (w *worker) release() { + w.taskQueue = nil + w.wg = nil + w.pool = nil +} + +// Pool defines coroutine pool interface +type Pool interface { + ShutDown() + Submit(t Task) (Future, error) + SubmitFunc(f func() interface{}) (Future, error) + Execute(t Task) + ExecuteFunc(f func() interface{}) + GetMaxWorkerCnt() int64 + AddMaxWorkerCnt(value int64) int64 + GetCurrentWorkingCnt() int64 + AddCurrentWorkingCnt(value int64) int64 + GetWorkerCnt() int64 + AddWorkerCnt(value int64) int64 + EnableAutoTune() +} + +type basicPool struct { + maxWorkerCnt int64 + workerCnt int64 + currentWorkingCnt int64 + isShutDown int32 +} + +// ErrTaskInvalid will be returned if the task is nil +var ErrTaskInvalid = errors.New("Task is nil") + +func (pool *basicPool) GetCurrentWorkingCnt() int64 { + return atomic.LoadInt64(&pool.currentWorkingCnt) +} + +func (pool *basicPool) AddCurrentWorkingCnt(value int64) int64 { + return atomic.AddInt64(&pool.currentWorkingCnt, value) +} + +func (pool *basicPool) GetWorkerCnt() int64 { + return atomic.LoadInt64(&pool.workerCnt) +} + +func (pool *basicPool) AddWorkerCnt(value int64) int64 { + return atomic.AddInt64(&pool.workerCnt, value) +} + +func (pool *basicPool) GetMaxWorkerCnt() int64 { + return atomic.LoadInt64(&pool.maxWorkerCnt) +} + +func (pool *basicPool) AddMaxWorkerCnt(value int64) int64 { + return atomic.AddInt64(&pool.maxWorkerCnt, value) +} + +func (pool *basicPool) CompareAndSwapCurrentWorkingCnt(oldValue, newValue int64) bool { + return atomic.CompareAndSwapInt64(&pool.currentWorkingCnt, oldValue, newValue) +} + +func (pool *basicPool) EnableAutoTune() { + +} + +// RoutinePool defines the coroutine pool struct +type RoutinePool struct { + basicPool + taskQueue chan Task + dispatchQueue chan Task + workers map[string]*worker + cacheCnt int + wg *sync.WaitGroup + lock *sync.Mutex + shutDownWg *sync.WaitGroup + autoTune int32 +} + +// ErrSubmitTimeout will be returned if submit task timeout when calling SubmitWithTimeout function +var ErrSubmitTimeout = errors.New("Submit task timeout") + +// ErrPoolShutDown will be returned if RoutinePool is shutdown +var ErrPoolShutDown = errors.New("RoutinePool is shutdown") + +// ErrTaskReject will be returned if submit task is rejected +var ErrTaskReject = errors.New("Submit task is rejected") + +var closeQueue = signalTask{id: "closeQueue"} + +// NewRoutinePool creates a RoutinePool instance +func NewRoutinePool(maxWorkerCnt, cacheCnt int) Pool { + if maxWorkerCnt <= 0 { + maxWorkerCnt = runtime.NumCPU() + } + + pool := &RoutinePool{ + cacheCnt: cacheCnt, + wg: new(sync.WaitGroup), + lock: new(sync.Mutex), + shutDownWg: new(sync.WaitGroup), + autoTune: 0, + } + pool.isShutDown = 0 + pool.maxWorkerCnt += int64(maxWorkerCnt) + if pool.cacheCnt <= 0 { + pool.taskQueue = make(chan Task) + } else { + pool.taskQueue = make(chan Task, pool.cacheCnt) + } + pool.workers = make(map[string]*worker, pool.maxWorkerCnt) + // dispatchQueue must not have length + pool.dispatchQueue = make(chan Task) + pool.dispatcher() + + return pool +} + +// EnableAutoTune sets the autoTune enabled +func (pool *RoutinePool) EnableAutoTune() { + atomic.StoreInt32(&pool.autoTune, 1) +} + +func (pool *RoutinePool) checkStatus(t Task) error { + if t == nil { + return ErrTaskInvalid + } + + if atomic.LoadInt32(&pool.isShutDown) == 1 { + return ErrPoolShutDown + } + return nil +} + +func (pool *RoutinePool) dispatcher() { + pool.shutDownWg.Add(1) + go func() { + for { + task, ok := <-pool.dispatchQueue + if !ok { + break + } + + if task == closeQueue { + close(pool.taskQueue) + pool.shutDownWg.Done() + continue + } + + if pool.GetWorkerCnt() < pool.GetMaxWorkerCnt() { + pool.addWorker() + } + + pool.taskQueue <- task + } + }() +} + +// AddMaxWorkerCnt sets the maxWorkerCnt field's value and returns it +func (pool *RoutinePool) AddMaxWorkerCnt(value int64) int64 { + if atomic.LoadInt32(&pool.autoTune) == 1 { + return pool.basicPool.AddMaxWorkerCnt(value) + } + return pool.GetMaxWorkerCnt() +} + +func (pool *RoutinePool) addWorker() { + if atomic.LoadInt32(&pool.autoTune) == 1 { + pool.lock.Lock() + defer pool.lock.Unlock() + } + w := &worker{} + w.name = fmt.Sprintf("woker-%d", len(pool.workers)) + w.taskQueue = pool.taskQueue + w.wg = pool.wg + pool.AddWorkerCnt(1) + w.pool = pool + pool.workers[w.name] = w + pool.wg.Add(1) + w.start() +} + +func (pool *RoutinePool) autoTuneWorker(w *worker) bool { + if atomic.LoadInt32(&pool.autoTune) == 0 { + return false + } + + if w == nil { + return false + } + + workerCnt := pool.GetWorkerCnt() + maxWorkerCnt := pool.GetMaxWorkerCnt() + if workerCnt > maxWorkerCnt && atomic.CompareAndSwapInt64(&pool.workerCnt, workerCnt, workerCnt-1) { + pool.lock.Lock() + defer pool.lock.Unlock() + delete(pool.workers, w.name) + w.wg.Done() + w.release() + return true + } + + return false +} + +// ExecuteFunc creates a funcWrapper instance with the specified function and calls the Execute function +func (pool *RoutinePool) ExecuteFunc(f func() interface{}) { + fw := &funcWrapper{ + f: f, + } + pool.Execute(fw) +} + +// Execute pushes the specified task to the dispatchQueue +func (pool *RoutinePool) Execute(t Task) { + if t != nil { + pool.dispatchQueue <- t + } +} + +// SubmitFunc creates a funcWrapper instance with the specified function and calls the Submit function +func (pool *RoutinePool) SubmitFunc(f func() interface{}) (Future, error) { + fw := &funcWrapper{ + f: f, + } + return pool.Submit(fw) +} + +// Submit pushes the specified task to the dispatchQueue, and returns the FutureResult and error info +func (pool *RoutinePool) Submit(t Task) (Future, error) { + if err := pool.checkStatus(t); err != nil { + return nil, err + } + f := &FutureResult{} + f.resultChan = make(chan interface{}, 1) + tw := &taskWrapper{ + t: t, + f: f, + } + pool.dispatchQueue <- tw + return f, nil +} + +// SubmitWithTimeout pushes the specified task to the dispatchQueue, and returns the FutureResult and error info. +// Also takes a timeout value, will return ErrSubmitTimeout if it does't complete within that time. +func (pool *RoutinePool) SubmitWithTimeout(t Task, timeout int64) (Future, error) { + if timeout <= 0 { + return pool.Submit(t) + } + if err := pool.checkStatus(t); err != nil { + return nil, err + } + timeoutChan := make(chan bool, 1) + go func() { + time.Sleep(time.Duration(time.Millisecond * time.Duration(timeout))) + timeoutChan <- true + close(timeoutChan) + }() + + f := &FutureResult{} + f.resultChan = make(chan interface{}, 1) + tw := &taskWrapper{ + t: t, + f: f, + } + select { + case pool.dispatchQueue <- tw: + return f, nil + case _, ok := <-timeoutChan: + if ok { + return nil, ErrSubmitTimeout + } + return nil, ErrSubmitTimeout + } +} + +func (pool *RoutinePool) beforeCloseDispatchQueue() { + if !atomic.CompareAndSwapInt32(&pool.isShutDown, 0, 1) { + return + } + pool.dispatchQueue <- closeQueue + pool.wg.Wait() +} + +func (pool *RoutinePool) doCloseDispatchQueue() { + close(pool.dispatchQueue) + pool.shutDownWg.Wait() +} + +// ShutDown closes the RoutinePool instance +func (pool *RoutinePool) ShutDown() { + pool.beforeCloseDispatchQueue() + pool.doCloseDispatchQueue() + for _, w := range pool.workers { + w.release() + } + pool.workers = nil + pool.taskQueue = nil + pool.dispatchQueue = nil +} + +// NoChanPool defines the coroutine pool struct +type NoChanPool struct { + basicPool + wg *sync.WaitGroup + tokens chan interface{} +} + +// NewNochanPool creates a new NoChanPool instance +func NewNochanPool(maxWorkerCnt int) Pool { + if maxWorkerCnt <= 0 { + maxWorkerCnt = runtime.NumCPU() + } + + pool := &NoChanPool{ + wg: new(sync.WaitGroup), + tokens: make(chan interface{}, maxWorkerCnt), + } + pool.isShutDown = 0 + pool.AddMaxWorkerCnt(int64(maxWorkerCnt)) + + for i := 0; i < maxWorkerCnt; i++ { + pool.tokens <- struct{}{} + } + + return pool +} + +func (pool *NoChanPool) acquire() { + <-pool.tokens +} + +func (pool *NoChanPool) release() { + pool.tokens <- 1 +} + +func (pool *NoChanPool) execute(t Task) { + pool.wg.Add(1) + go func() { + pool.acquire() + defer func() { + pool.release() + pool.wg.Done() + }() + runTask(t) + }() +} + +// ShutDown closes the NoChanPool instance +func (pool *NoChanPool) ShutDown() { + if !atomic.CompareAndSwapInt32(&pool.isShutDown, 0, 1) { + return + } + pool.wg.Wait() +} + +// Execute executes the specified task +func (pool *NoChanPool) Execute(t Task) { + if t != nil { + pool.execute(t) + } +} + +// ExecuteFunc creates a funcWrapper instance with the specified function and calls the Execute function +func (pool *NoChanPool) ExecuteFunc(f func() interface{}) { + fw := &funcWrapper{ + f: f, + } + pool.Execute(fw) +} + +// Submit executes the specified task, and returns the FutureResult and error info +func (pool *NoChanPool) Submit(t Task) (Future, error) { + if t == nil { + return nil, ErrTaskInvalid + } + + f := &FutureResult{} + f.resultChan = make(chan interface{}, 1) + tw := &taskWrapper{ + t: t, + f: f, + } + + pool.execute(tw) + return f, nil +} + +// SubmitFunc creates a funcWrapper instance with the specified function and calls the Submit function +func (pool *NoChanPool) SubmitFunc(f func() interface{}) (Future, error) { + fw := &funcWrapper{ + f: f, + } + return pool.Submit(fw) +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/provider.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/provider.go new file mode 100644 index 0000000000..2e485c1134 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/provider.go @@ -0,0 +1,243 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "encoding/json" + "io/ioutil" + "math/rand" + "net" + "net/http" + "os" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + accessKeyEnv = "OBS_ACCESS_KEY_ID" + securityKeyEnv = "OBS_SECRET_ACCESS_KEY" + securityTokenEnv = "OBS_SECURITY_TOKEN" + ecsRequestURL = "http://169.254.169.254/openstack/latest/securitykey" +) + +type securityHolder struct { + ak string + sk string + securityToken string +} + +var emptySecurityHolder = securityHolder{} + +type securityProvider interface { + getSecurity() securityHolder +} + +type BasicSecurityProvider struct { + val atomic.Value +} + +func (bsp *BasicSecurityProvider) getSecurity() securityHolder { + if sh, ok := bsp.val.Load().(securityHolder); ok { + return sh + } + return emptySecurityHolder +} + +func (bsp *BasicSecurityProvider) refresh(ak, sk, securityToken string) { + bsp.val.Store(securityHolder{ak: strings.TrimSpace(ak), sk: strings.TrimSpace(sk), securityToken: strings.TrimSpace(securityToken)}) +} + +func NewBasicSecurityProvider(ak, sk, securityToken string) *BasicSecurityProvider { + bsp := &BasicSecurityProvider{} + bsp.refresh(ak, sk, securityToken) + return bsp +} + +type EnvSecurityProvider struct { + sh securityHolder + suffix string + once sync.Once +} + +func (esp *EnvSecurityProvider) getSecurity() securityHolder { + //ensure run only once + esp.once.Do(func() { + esp.sh = securityHolder{ + ak: strings.TrimSpace(os.Getenv(accessKeyEnv + esp.suffix)), + sk: strings.TrimSpace(os.Getenv(securityKeyEnv + esp.suffix)), + securityToken: strings.TrimSpace(os.Getenv(securityTokenEnv + esp.suffix)), + } + }) + + return esp.sh +} + +func NewEnvSecurityProvider(suffix string) *EnvSecurityProvider { + if suffix != "" { + suffix = "_" + suffix + } + esp := &EnvSecurityProvider{ + suffix: suffix, + } + return esp +} + +type TemporarySecurityHolder struct { + securityHolder + expireDate time.Time +} + +var emptyTemporarySecurityHolder = TemporarySecurityHolder{} + +type EcsSecurityProvider struct { + val atomic.Value + lock sync.Mutex + httpClient *http.Client + prefetch int32 + retryCount int +} + +func (ecsSp *EcsSecurityProvider) loadTemporarySecurityHolder() (TemporarySecurityHolder, bool) { + if sh := ecsSp.val.Load(); sh == nil { + return emptyTemporarySecurityHolder, false + } else if _sh, ok := sh.(TemporarySecurityHolder); !ok { + return emptyTemporarySecurityHolder, false + } else { + return _sh, true + } +} + +func (ecsSp *EcsSecurityProvider) getAndSetSecurityWithOutLock() securityHolder { + _sh := TemporarySecurityHolder{} + _sh.expireDate = time.Now().Add(time.Minute * 5) + retryCount := 0 + for { + if req, err := http.NewRequest("GET", ecsRequestURL, nil); err == nil { + start := GetCurrentTimestamp() + res, err := ecsSp.httpClient.Do(req) + if err == nil { + if data, _err := ioutil.ReadAll(res.Body); _err == nil { + temp := &struct { + Credential struct { + AK string `json:"access,omitempty"` + SK string `json:"secret,omitempty"` + SecurityToken string `json:"securitytoken,omitempty"` + ExpireDate time.Time `json:"expires_at,omitempty"` + } `json:"credential"` + }{} + + doLog(LEVEL_DEBUG, "Get the json data from ecs succeed") + + if jsonErr := json.Unmarshal(data, temp); jsonErr == nil { + _sh.ak = temp.Credential.AK + _sh.sk = temp.Credential.SK + _sh.securityToken = temp.Credential.SecurityToken + _sh.expireDate = temp.Credential.ExpireDate.Add(time.Minute * -1) + + doLog(LEVEL_INFO, "Get security from ecs succeed, AK:xxxx, SK:xxxx, SecurityToken:xxxx, ExprireDate %s", _sh.expireDate) + + doLog(LEVEL_INFO, "Get security from ecs succeed, cost %d ms", (GetCurrentTimestamp() - start)) + break + } else { + err = jsonErr + } + } else { + err = _err + } + } + + doLog(LEVEL_WARN, "Try to get security from ecs failed, cost %d ms, err %s", (GetCurrentTimestamp() - start), err.Error()) + } + + if retryCount >= ecsSp.retryCount { + doLog(LEVEL_WARN, "Try to get security from ecs failed and exceed the max retry count") + break + } + sleepTime := float64(retryCount+2) * rand.Float64() + if sleepTime > 10 { + sleepTime = 10 + } + time.Sleep(time.Duration(sleepTime * float64(time.Second))) + retryCount++ + } + + ecsSp.val.Store(_sh) + return _sh.securityHolder +} + +func (ecsSp *EcsSecurityProvider) getAndSetSecurity() securityHolder { + ecsSp.lock.Lock() + defer ecsSp.lock.Unlock() + tsh, succeed := ecsSp.loadTemporarySecurityHolder() + if !succeed || time.Now().After(tsh.expireDate) { + return ecsSp.getAndSetSecurityWithOutLock() + } + return tsh.securityHolder +} + +func (ecsSp *EcsSecurityProvider) getSecurity() securityHolder { + if tsh, succeed := ecsSp.loadTemporarySecurityHolder(); succeed { + if time.Now().Before(tsh.expireDate) { + //not expire + if time.Now().Add(time.Minute*5).After(tsh.expireDate) && atomic.CompareAndSwapInt32(&ecsSp.prefetch, 0, 1) { + //do prefetch + sh := ecsSp.getAndSetSecurityWithOutLock() + atomic.CompareAndSwapInt32(&ecsSp.prefetch, 1, 0) + return sh + } + return tsh.securityHolder + } + return ecsSp.getAndSetSecurity() + } + + return ecsSp.getAndSetSecurity() +} + +func getInternalTransport() *http.Transport { + + timeout := 10 + transport := &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { + start := GetCurrentTimestamp() + conn, err := (&net.Dialer{ + Timeout: time.Second * time.Duration(timeout), + Resolver: net.DefaultResolver, + }).Dial(network, addr) + + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http dial cost %d ms", (GetCurrentTimestamp() - start)) + } + if err != nil { + return nil, err + } + return getConnDelegate(conn, timeout, timeout*10), nil + }, + MaxIdleConns: 10, + MaxIdleConnsPerHost: 10, + ResponseHeaderTimeout: time.Second * time.Duration(timeout), + IdleConnTimeout: time.Second * time.Duration(DEFAULT_IDLE_CONN_TIMEOUT), + DisableCompression: true, + } + + return transport +} + +func NewEcsSecurityProvider(retryCount int) *EcsSecurityProvider { + ecsSp := &EcsSecurityProvider{ + retryCount: retryCount, + } + ecsSp.httpClient = &http.Client{Transport: getInternalTransport(), CheckRedirect: checkRedirectFunc} + return ecsSp +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_createSignedUrl.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_createSignedUrl.go new file mode 100644 index 0000000000..bb0eb8433c --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_createSignedUrl.go @@ -0,0 +1,53 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" +) + +// CreateSignedUrl creates signed url with the specified CreateSignedUrlInput, and returns the CreateSignedUrlOutput and error +func (obsClient ObsClient) CreateSignedUrl(input *CreateSignedUrlInput) (output *CreateSignedUrlOutput, err error) { + if input == nil { + return nil, errors.New("CreateSignedUrlInput is nil") + } + + params := make(map[string]string, len(input.QueryParams)) + for key, value := range input.QueryParams { + params[key] = value + } + + if input.SubResource != "" { + params[string(input.SubResource)] = "" + } + + headers := make(map[string][]string, len(input.Headers)) + for key, value := range input.Headers { + headers[key] = []string{value} + } + + if input.Expires <= 0 { + input.Expires = 300 + } + + requestURL, err := obsClient.doAuthTemporary(string(input.Method), input.Bucket, input.Key, params, headers, int64(input.Expires)) + if err != nil { + return nil, err + } + + output = &CreateSignedUrlOutput{ + SignedUrl: requestURL, + ActualSignedRequestHeaders: headers, + } + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_other.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_other.go new file mode 100644 index 0000000000..d2ae13f72f --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_other.go @@ -0,0 +1,116 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" + "fmt" + "strings" + "time" +) + +func (obsClient ObsClient) isSecurityToken(params map[string]string, sh securityHolder) { + if sh.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = sh.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = sh.securityToken + } + } +} + +// CreateBrowserBasedSignature gets the browser based signature with the specified CreateBrowserBasedSignatureInput, +// and returns the CreateBrowserBasedSignatureOutput and error +func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBasedSignatureInput) (output *CreateBrowserBasedSignatureOutput, err error) { + if input == nil { + return nil, errors.New("CreateBrowserBasedSignatureInput is nil") + } + + params := make(map[string]string, len(input.FormParams)) + for key, value := range input.FormParams { + params[key] = value + } + + date := time.Now().UTC() + shortDate := date.Format(SHORT_DATE_FORMAT) + longDate := date.Format(LONG_DATE_FORMAT) + sh := obsClient.getSecurity() + + credential, _ := getCredential(sh.ak, obsClient.conf.region, shortDate) + + if input.Expires <= 0 { + input.Expires = 300 + } + + expiration := date.Add(time.Second * time.Duration(input.Expires)).Format(ISO8601_DATE_FORMAT) + if obsClient.conf.signature == SignatureV4 { + params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX + params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + params[PARAM_DATE_AMZ_CAMEL] = longDate + } + + obsClient.isSecurityToken(params, sh) + + matchAnyBucket := true + matchAnyKey := true + count := 5 + if bucket := strings.TrimSpace(input.Bucket); bucket != "" { + params["bucket"] = bucket + matchAnyBucket = false + count-- + } + + if key := strings.TrimSpace(input.Key); key != "" { + params["key"] = key + matchAnyKey = false + count-- + } + + originPolicySlice := make([]string, 0, len(params)+count) + originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"expiration\":\"%s\",", expiration)) + originPolicySlice = append(originPolicySlice, "\"conditions\":[") + for key, value := range params { + if _key := strings.TrimSpace(strings.ToLower(key)); _key != "" { + originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"%s\":\"%s\"},", _key, value)) + } + } + + if matchAnyBucket { + originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$bucket\", \"\"],") + } + + if matchAnyKey { + originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$key\", \"\"],") + } + + originPolicySlice = append(originPolicySlice, "]}") + + originPolicy := strings.Join(originPolicySlice, "") + policy := Base64Encode([]byte(originPolicy)) + var signature string + if obsClient.conf.signature == SignatureV4 { + signature = getSignature(policy, sh.sk, obsClient.conf.region, shortDate) + } else { + signature = Base64Encode(HmacSha1([]byte(sh.sk), []byte(policy))) + } + + output = &CreateBrowserBasedSignatureOutput{ + OriginPolicy: originPolicy, + Policy: policy, + Algorithm: params[PARAM_ALGORITHM_AMZ_CAMEL], + Credential: params[PARAM_CREDENTIAL_AMZ_CAMEL], + Date: params[PARAM_DATE_AMZ_CAMEL], + Signature: signature, + } + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_signedUrl.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_signedUrl.go new file mode 100644 index 0000000000..be927829f0 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/temporary_signedUrl.go @@ -0,0 +1,758 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "errors" + "io" + "net/http" + "os" + "strings" +) + +// ListBucketsWithSignedUrl lists buckets with the specified signed url and signed request headers +func (obsClient ObsClient) ListBucketsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListBucketsOutput, err error) { + output = &ListBucketsOutput{} + err = obsClient.doHTTPWithSignedURL("ListBuckets", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// CreateBucketWithSignedUrl creates bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) CreateBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("CreateBucket", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketWithSignedUrl deletes bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucket", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketStoragePolicyWithSignedUrl sets bucket storage class with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketStoragePolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketStoragePolicyWithSignedUrl gets bucket storage class with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketStoragePolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// ListObjectsWithSignedUrl lists objects in a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) ListObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListObjectsOutput, err error) { + output = &ListObjectsOutput{} + err = obsClient.doHTTPWithSignedURL("ListObjects", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + if output.EncodingType == "url" { + err = decodeListObjectsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListObjectsOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// ListVersionsWithSignedUrl lists versioning objects in a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) ListVersionsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListVersionsOutput, err error) { + output = &ListVersionsOutput{} + err = obsClient.doHTTPWithSignedURL("ListVersions", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + if output.EncodingType == "url" { + err = decodeListVersionsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListVersionsOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// ListMultipartUploadsWithSignedUrl lists the multipart uploads that are initialized but not combined or aborted in a +// specified bucket with the specified signed url and signed request headers +func (obsClient ObsClient) ListMultipartUploadsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListMultipartUploadsOutput, err error) { + output = &ListMultipartUploadsOutput{} + err = obsClient.doHTTPWithSignedURL("ListMultipartUploads", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else if output.EncodingType == "url" { + err = decodeListMultipartUploadsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListMultipartUploadsOutput with error: %v.", err) + output = nil + } + } + return +} + +// SetBucketQuotaWithSignedUrl sets the bucket quota with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketQuota", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketQuotaWithSignedUrl gets the bucket quota with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketQuotaOutput, err error) { + output = &GetBucketQuotaOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketQuota", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// HeadBucketWithSignedUrl checks whether a bucket exists with the specified signed url and signed request headers +func (obsClient ObsClient) HeadBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("HeadBucket", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// HeadObjectWithSignedUrl checks whether an object exists with the specified signed url and signed request headers +func (obsClient ObsClient) HeadObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("HeadObject", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketMetadataWithSignedUrl gets the metadata of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketMetadataOutput, err error) { + output = &GetBucketMetadataOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetBucketMetadataOutput(output) + } + return +} + +// GetBucketStorageInfoWithSignedUrl gets storage information about a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketStorageInfoWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStorageInfoOutput, err error) { + output = &GetBucketStorageInfoOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketStorageInfo", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketLocationWithSignedUrl gets the location of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketLocationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketLocation", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketAclWithSignedUrl sets the bucket ACL with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketAclWithSignedUrl gets the bucket ACL with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketPolicyWithSignedUrl sets the bucket policy with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketPolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketPolicyWithSignedUrl gets the bucket policy with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketPolicyOutput, err error) { + output = &GetBucketPolicyOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketPolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, false) + if err != nil { + output = nil + } + return +} + +// DeleteBucketPolicyWithSignedUrl deletes the bucket policy with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketPolicy", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketCorsWithSignedUrl sets CORS rules for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketCors", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketCorsWithSignedUrl gets CORS rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketCorsOutput, err error) { + output = &GetBucketCorsOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketCors", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketCorsWithSignedUrl deletes CORS rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketCors", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketVersioningWithSignedUrl sets the versioning status for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketVersioning", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketVersioningWithSignedUrl gets the versioning status of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketVersioningOutput, err error) { + output = &GetBucketVersioningOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketVersioning", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketWebsiteConfigurationWithSignedUrl sets website hosting for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketWebsiteConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketWebsiteConfigurationWithSignedUrl gets the website hosting settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketWebsiteConfigurationOutput, err error) { + output = &GetBucketWebsiteConfigurationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketWebsiteConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketWebsiteConfigurationWithSignedUrl deletes the website hosting settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketWebsiteConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketLoggingConfigurationWithSignedUrl sets the bucket logging with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketLoggingConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketLoggingConfigurationWithSignedUrl gets the logging settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLoggingConfigurationOutput, err error) { + output = &GetBucketLoggingConfigurationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketLoggingConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketLifecycleConfigurationWithSignedUrl sets lifecycle rules for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketLifecycleConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketLifecycleConfigurationWithSignedUrl gets lifecycle rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLifecycleConfigurationOutput, err error) { + output = &GetBucketLifecycleConfigurationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketLifecycleConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketLifecycleConfigurationWithSignedUrl deletes lifecycle rules of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketLifecycleConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketTaggingWithSignedUrl sets bucket tags with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketTagging", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketTaggingWithSignedUrl gets bucket tags with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketTaggingOutput, err error) { + output = &GetBucketTaggingOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketTagging", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketTaggingWithSignedUrl deletes bucket tags with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketTagging", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketNotificationWithSignedUrl sets event notification for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketNotification", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketNotificationWithSignedUrl gets event notification settings of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketNotificationOutput, err error) { + output = &GetBucketNotificationOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketNotification", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteObjectWithSignedUrl deletes an object with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *DeleteObjectOutput, err error) { + output = &DeleteObjectOutput{} + err = obsClient.doHTTPWithSignedURL("DeleteObject", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseDeleteObjectOutput(output) + } + return +} + +// DeleteObjectsWithSignedUrl deletes objects in a batch with the specified signed url and signed request headers and data +func (obsClient ObsClient) DeleteObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *DeleteObjectsOutput, err error) { + output = &DeleteObjectsOutput{} + err = obsClient.doHTTPWithSignedURL("DeleteObjects", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else if output.EncodingType == "url" { + err = decodeDeleteObjectsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get DeleteObjectsOutput with error: %v.", err) + output = nil + } + } + return +} + +// SetObjectAclWithSignedUrl sets ACL for an object with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetObjectAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetObjectAclWithSignedUrl gets the ACL of an object with the specified signed url and signed request headers +func (obsClient ObsClient) GetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectAclOutput, err error) { + output = &GetObjectAclOutput{} + err = obsClient.doHTTPWithSignedURL("GetObjectAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if versionID, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionID[0] + } + } + return +} + +// RestoreObjectWithSignedUrl restores an object with the specified signed url and signed request headers and data +func (obsClient ObsClient) RestoreObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("RestoreObject", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetObjectMetadataWithSignedUrl gets object metadata with the specified signed url and signed request headers +func (obsClient ObsClient) GetObjectMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectMetadataOutput, err error) { + output = &GetObjectMetadataOutput{} + err = obsClient.doHTTPWithSignedURL("GetObjectMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetObjectMetadataOutput(output) + } + return +} + +// GetObjectWithSignedUrl downloads object with the specified signed url and signed request headers +func (obsClient ObsClient) GetObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectOutput, err error) { + output = &GetObjectOutput{} + err = obsClient.doHTTPWithSignedURL("GetObject", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetObjectOutput(output) + } + return +} + +// PutObjectWithSignedUrl uploads an object to the specified bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) PutObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *PutObjectOutput, err error) { + output = &PutObjectOutput{} + err = obsClient.doHTTPWithSignedURL("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +// PutFileWithSignedUrl uploads a file to the specified bucket with the specified signed url and signed request headers and sourceFile path +func (obsClient ObsClient) PutFileWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, sourceFile string) (output *PutObjectOutput, err error) { + var data io.Reader + sourceFile = strings.TrimSpace(sourceFile) + if sourceFile != "" { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err + return nil, err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() + + stat, _err := fd.Stat() + if _err != nil { + err = _err + return nil, err + } + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + + var contentLength int64 + if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH_CAMEL]; ok { + contentLength = StringToInt64(value[0], -1) + } else if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH]; ok { + contentLength = StringToInt64(value[0], -1) + } else { + contentLength = stat.Size() + } + if contentLength > stat.Size() { + return nil, errors.New("ContentLength is larger than fileSize") + } + fileReaderWrapper.totalCount = contentLength + data = fileReaderWrapper + } + + output = &PutObjectOutput{} + err = obsClient.doHTTPWithSignedURL("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +// CopyObjectWithSignedUrl creates a copy for an existing object with the specified signed url and signed request headers +func (obsClient ObsClient) CopyObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyObjectOutput, err error) { + output = &CopyObjectOutput{} + err = obsClient.doHTTPWithSignedURL("CopyObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseCopyObjectOutput(output) + } + return +} + +// AbortMultipartUploadWithSignedUrl aborts a multipart upload in a specified bucket by using the multipart upload ID with the specified signed url and signed request headers +func (obsClient ObsClient) AbortMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("AbortMultipartUpload", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// InitiateMultipartUploadWithSignedUrl initializes a multipart upload with the specified signed url and signed request headers +func (obsClient ObsClient) InitiateMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *InitiateMultipartUploadOutput, err error) { + output = &InitiateMultipartUploadOutput{} + err = obsClient.doHTTPWithSignedURL("InitiateMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseInitiateMultipartUploadOutput(output) + if output.EncodingType == "url" { + err = decodeInitiateMultipartUploadOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get InitiateMultipartUploadOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// UploadPartWithSignedUrl uploads a part to a specified bucket by using a specified multipart upload ID +// with the specified signed url and signed request headers and data +func (obsClient ObsClient) UploadPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *UploadPartOutput, err error) { + output = &UploadPartOutput{} + err = obsClient.doHTTPWithSignedURL("UploadPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParseUploadPartOutput(output) + } + return +} + +// CompleteMultipartUploadWithSignedUrl combines the uploaded parts in a specified bucket by using the multipart upload ID +// with the specified signed url and signed request headers and data +func (obsClient ObsClient) CompleteMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *CompleteMultipartUploadOutput, err error) { + output = &CompleteMultipartUploadOutput{} + err = obsClient.doHTTPWithSignedURL("CompleteMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParseCompleteMultipartUploadOutput(output) + if output.EncodingType == "url" { + err = decodeCompleteMultipartUploadOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get CompleteMultipartUploadOutput with error: %v.", err) + output = nil + } + } + } + return +} + +// ListPartsWithSignedUrl lists the uploaded parts in a bucket by using the multipart upload ID with the specified signed url and signed request headers +func (obsClient ObsClient) ListPartsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListPartsOutput, err error) { + output = &ListPartsOutput{} + err = obsClient.doHTTPWithSignedURL("ListParts", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else if output.EncodingType == "url" { + err = decodeListPartsOutput(output) + if err != nil { + doLog(LEVEL_ERROR, "Failed to get ListPartsOutput with error: %v.", err) + output = nil + } + } + return +} + +// CopyPartWithSignedUrl copy a part to a specified bucket by using a specified multipart upload ID with the specified signed url and signed request headers +func (obsClient ObsClient) CopyPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyPartOutput, err error) { + output = &CopyPartOutput{} + err = obsClient.doHTTPWithSignedURL("CopyPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseCopyPartOutput(output) + } + return +} + +// SetBucketRequestPaymentWithSignedUrl sets requester-pays setting for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketRequestPaymentWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketRequestPayment", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketRequestPaymentWithSignedUrl gets requester-pays setting of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketRequestPaymentWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketRequestPaymentOutput, err error) { + output = &GetBucketRequestPaymentOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketRequestPayment", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// SetBucketEncryptionWithSignedURL sets bucket encryption setting for a bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) SetBucketEncryptionWithSignedURL(signedURL string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("SetBucketEncryption", HTTP_PUT, signedURL, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +// GetBucketEncryptionWithSignedURL gets bucket encryption setting of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) GetBucketEncryptionWithSignedURL(signedURL string, actualSignedRequestHeaders http.Header) (output *GetBucketEncryptionOutput, err error) { + output = &GetBucketEncryptionOutput{} + err = obsClient.doHTTPWithSignedURL("GetBucketEncryption", HTTP_GET, signedURL, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// DeleteBucketEncryptionWithSignedURL deletes bucket encryption setting of a bucket with the specified signed url and signed request headers +func (obsClient ObsClient) DeleteBucketEncryptionWithSignedURL(signedURL string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHTTPWithSignedURL("DeleteBucketEncryption", HTTP_DELETE, signedURL, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +// AppendObjectWithSignedUrl uploads an object to the specified bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) AppendObjectWithSignedURL(signedURL string, actualSignedRequestHeaders http.Header, data io.Reader) (output *AppendObjectOutput, err error) { + output = &AppendObjectOutput{} + err = obsClient.doHTTPWithSignedURL("AppendObject", HTTP_POST, signedURL, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + if err = ParseAppendObjectOutput(output); err != nil { + output = nil + } + } + return +} + +// ModifyObjectWithSignedUrl uploads an object to the specified bucket with the specified signed url and signed request headers and data +func (obsClient ObsClient) ModifyObjectWithSignedURL(signedURL string, actualSignedRequestHeaders http.Header, data io.Reader) (output *ModifyObjectOutput, err error) { + output = &ModifyObjectOutput{} + err = obsClient.doHTTPWithSignedURL("ModifyObject", HTTP_PUT, signedURL, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParseModifyObjectOutput(output) + } + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/trait.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/trait.go new file mode 100644 index 0000000000..203fd3923e --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/trait.go @@ -0,0 +1,968 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +//nolint:structcheck, unused +package obs + +import ( + "bytes" + "fmt" + "io" + "net/url" + "os" + "strconv" + "strings" +) + +// IReadCloser defines interface with function: setReadCloser +type IReadCloser interface { + setReadCloser(body io.ReadCloser) +} + +func (output *GetObjectOutput) setReadCloser(body io.ReadCloser) { + output.Body = body +} + +func setHeaders(headers map[string][]string, header string, headerValue []string, isObs bool) { + if isObs { + header = HEADER_PREFIX_OBS + header + headers[header] = headerValue + } else { + header = HEADER_PREFIX + header + headers[header] = headerValue + } +} + +func setHeadersNext(headers map[string][]string, header string, headerNext string, headerValue []string, isObs bool) { + if isObs { + headers[header] = headerValue + } else { + headers[headerNext] = headerValue + } +} + +// IBaseModel defines interface for base response model +type IBaseModel interface { + setStatusCode(statusCode int) + + setRequestID(requestID string) + + setResponseHeaders(responseHeaders map[string][]string) +} + +// ISerializable defines interface with function: trans +type ISerializable interface { + trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) +} + +// DefaultSerializable defines default serializable struct +type DefaultSerializable struct { + params map[string]string + headers map[string][]string + data interface{} +} + +func (input DefaultSerializable) trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) { + return input.params, input.headers, input.data, nil +} + +var defaultSerializable = &DefaultSerializable{} + +func newSubResourceSerial(subResource SubResourceType) *DefaultSerializable { + return &DefaultSerializable{map[string]string{string(subResource): ""}, nil, nil} +} + +func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(subResource): ""} + data, err = ConvertRequestToIoReader(input) + return +} + +func (baseModel *BaseModel) setStatusCode(statusCode int) { + baseModel.StatusCode = statusCode +} + +func (baseModel *BaseModel) setRequestID(requestID string) { + baseModel.RequestId = requestID +} + +func (baseModel *BaseModel) setResponseHeaders(responseHeaders map[string][]string) { + baseModel.ResponseHeaders = responseHeaders +} + +func (input ListBucketsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + if input.QueryLocation && !isObs { + setHeaders(headers, HEADER_LOCATION_AMZ, []string{"true"}, isObs) + } + if input.BucketType != "" { + setHeaders(headers, HEADER_BUCKET_TYPE, []string{string(input.BucketType)}, true) + } + return +} + +func (input CreateBucketInput) prepareGrantHeaders(headers map[string][]string, isObs bool) { + if grantReadID := input.GrantReadId; grantReadID != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{grantReadID}, isObs) + } + if grantWriteID := input.GrantWriteId; grantWriteID != "" { + setHeaders(headers, HEADER_GRANT_WRITE_OBS, []string{grantWriteID}, isObs) + } + if grantReadAcpID := input.GrantReadAcpId; grantReadAcpID != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{grantReadAcpID}, isObs) + } + if grantWriteAcpID := input.GrantWriteAcpId; grantWriteAcpID != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{grantWriteAcpID}, isObs) + } + if grantFullControlID := input.GrantFullControlId; grantFullControlID != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{grantFullControlID}, isObs) + } + if grantReadDeliveredID := input.GrantReadDeliveredId; grantReadDeliveredID != "" { + setHeaders(headers, HEADER_GRANT_READ_DELIVERED_OBS, []string{grantReadDeliveredID}, true) + } + if grantFullControlDeliveredID := input.GrantFullControlDeliveredId; grantFullControlDeliveredID != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS, []string{grantFullControlDeliveredID}, true) + } +} + +func (input CreateBucketInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + setHeadersNext(headers, HEADER_STORAGE_CLASS_OBS, HEADER_STORAGE_CLASS, []string{storageClass}, isObs) + } + if epid := input.Epid; epid != "" { + setHeaders(headers, HEADER_EPID_HEADERS, []string{epid}, isObs) + } + if availableZone := input.AvailableZone; availableZone != "" { + setHeaders(headers, HEADER_AZ_REDUNDANCY, []string{availableZone}, isObs) + } + + input.prepareGrantHeaders(headers, isObs) + if input.IsFSFileInterface { + setHeaders(headers, headerFSFileInterface, []string{"Enabled"}, true) + } + + if location := strings.TrimSpace(input.Location); location != "" { + input.Location = location + + xml := make([]string, 0, 3) + xml = append(xml, "") + if isObs { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } else { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } + xml = append(xml, "") + + data = strings.Join(xml, "") + } + return +} + +func (input SetBucketStoragePolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + xml := make([]string, 0, 1) + if !isObs { + storageClass := "STANDARD" + if input.StorageClass == StorageClassWarm { + storageClass = string(storageClassStandardIA) + } else if input.StorageClass == StorageClassCold { + storageClass = string(storageClassGlacier) + } + params = map[string]string{string(SubResourceStoragePolicy): ""} + xml = append(xml, fmt.Sprintf("%s", storageClass)) + } else { + if input.StorageClass != StorageClassWarm && input.StorageClass != StorageClassCold { + input.StorageClass = StorageClassStandard + } + params = map[string]string{string(SubResourceStorageClass): ""} + xml = append(xml, fmt.Sprintf("%s", input.StorageClass)) + } + data = strings.Join(xml, "") + return +} + +func (input ListObjsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.Prefix != "" { + params["prefix"] = input.Prefix + } + if input.Delimiter != "" { + params["delimiter"] = input.Delimiter + } + if input.MaxKeys > 0 { + params["max-keys"] = IntToString(input.MaxKeys) + } + if input.EncodingType != "" { + params["encoding-type"] = input.EncodingType + } + headers = make(map[string][]string) + if origin := strings.TrimSpace(input.Origin); origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{origin} + } + if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader} + } + return +} + +func (input ListObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } + if input.Marker != "" { + params["marker"] = input.Marker + } + return +} + +func (input ListVersionsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } + params[string(SubResourceVersions)] = "" + if input.KeyMarker != "" { + params["key-marker"] = input.KeyMarker + } + if input.VersionIdMarker != "" { + params["version-id-marker"] = input.VersionIdMarker + } + return +} + +func (input ListMultipartUploadsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceUploads): ""} + if input.Prefix != "" { + params["prefix"] = input.Prefix + } + if input.Delimiter != "" { + params["delimiter"] = input.Delimiter + } + if input.MaxUploads > 0 { + params["max-uploads"] = IntToString(input.MaxUploads) + } + if input.KeyMarker != "" { + params["key-marker"] = input.KeyMarker + } + if input.UploadIdMarker != "" { + params["upload-id-marker"] = input.UploadIdMarker + } + if input.EncodingType != "" { + params["encoding-type"] = input.EncodingType + } + return +} + +func (input SetBucketQuotaInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + return trans(SubResourceQuota, input) +} + +func (input SetBucketAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceAcl): ""} + headers = make(map[string][]string) + + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } else { + data, _ = convertBucketACLToXML(input.AccessControlPolicy, false, isObs) + } + return +} + +func (input SetBucketPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourcePolicy): ""} + data = strings.NewReader(input.Policy) + return +} + +func (input SetBucketCorsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceCors): ""} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketVersioningInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + return trans(SubResourceVersioning, input) +} + +func (input SetBucketWebsiteConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceWebsite): ""} + data, _ = ConvertWebsiteConfigurationToXml(input.BucketWebsiteConfiguration, false) + return +} + +func (input GetBucketMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + if origin := strings.TrimSpace(input.Origin); origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{origin} + } + if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader} + } + return +} + +func (input SetBucketLoggingConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceLogging): ""} + data, _ = ConvertLoggingStatusToXml(input.BucketLoggingStatus, false, isObs) + return +} + +func (input SetBucketLifecycleConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceLifecycle): ""} + data, md5 := ConvertLifecyleConfigurationToXml(input.BucketLifecyleConfiguration, true, isObs) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketEncryptionInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceEncryption): ""} + data, _ = ConvertEncryptionConfigurationToXml(input.BucketEncryptionConfiguration, false, isObs) + return +} + +func (input SetBucketTaggingInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceTagging): ""} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketNotificationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceNotification): ""} + data, _ = ConvertNotificationToXml(input.BucketNotification, false, isObs) + return +} + +func (input DeleteObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input DeleteObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceDelete): ""} + if strings.ToLower(input.EncodingType) == "url" { + for index, object := range input.Objects { + input.Objects[index].Key = url.QueryEscape(object.Key) + } + } + data, md5 := convertDeleteObjectsToXML(input) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceAcl): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } else { + data, _ = ConvertAclToXml(input.AccessControlPolicy, false, isObs) + } + return +} + +func (input GetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceAcl): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input RestoreObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{string(SubResourceRestore): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + if !isObs { + data, err = ConvertRequestToIoReader(input) + } else { + data = ConverntObsRestoreToXml(input) + } + return +} + +// GetEncryption gets the Encryption field value from SseKmsHeader +func (header SseKmsHeader) GetEncryption() string { + if header.Encryption != "" { + return header.Encryption + } + if !header.isObs { + return DEFAULT_SSE_KMS_ENCRYPTION + } + return DEFAULT_SSE_KMS_ENCRYPTION_OBS +} + +// GetKey gets the Key field value from SseKmsHeader +func (header SseKmsHeader) GetKey() string { + return header.Key +} + +// GetEncryption gets the Encryption field value from SseCHeader +func (header SseCHeader) GetEncryption() string { + if header.Encryption != "" { + return header.Encryption + } + return DEFAULT_SSE_C_ENCRYPTION +} + +// GetKey gets the Key field value from SseCHeader +func (header SseCHeader) GetKey() string { + return header.Key +} + +// GetKeyMD5 gets the KeyMD5 field value from SseCHeader +func (header SseCHeader) GetKeyMD5() string { + if header.KeyMD5 != "" { + return header.KeyMD5 + } + + if ret, err := Base64Decode(header.GetKey()); err == nil { + return Base64Md5(ret) + } + return "" +} + +func setSseHeader(headers map[string][]string, sseHeader ISseHeader, sseCOnly bool, isObs bool) { + if sseHeader != nil { + if sseCHeader, ok := sseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } else if sseKmsHeader, ok := sseHeader.(SseKmsHeader); !sseCOnly && ok { + sseKmsHeader.isObs = isObs + setHeaders(headers, HEADER_SSEKMS_ENCRYPTION, []string{sseKmsHeader.GetEncryption()}, isObs) + if sseKmsHeader.GetKey() != "" { + setHeadersNext(headers, HEADER_SSEKMS_KEY_OBS, HEADER_SSEKMS_KEY_AMZ, []string{sseKmsHeader.GetKey()}, isObs) + } + } + } +} + +func (input GetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + + if input.Origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{input.Origin} + } + + if input.RequestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{input.RequestHeader} + } + setSseHeader(headers, input.SseHeader, true, isObs) + return +} + +func (input SetObjectMetadataInput) prepareContentHeaders(headers map[string][]string) { + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION_CAMEL] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING_CAMEL] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE_CAMEL] = []string{input.ContentLanguage} + } + + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } +} + +func (input SetObjectMetadataInput) prepareStorageClass(headers map[string][]string, isObs bool) { + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) + } +} + +func (input SetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + params = map[string]string{string(SubResourceMetadata): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + + if directive := string(input.MetadataDirective); directive != "" { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(input.MetadataDirective)}, isObs) + } else { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(ReplaceNew)}, isObs) + } + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL_CAMEL] = []string{input.CacheControl} + } + input.prepareContentHeaders(headers) + if input.Expires != "" { + headers[HEADER_EXPIRES_CAMEL] = []string{input.Expires} + } + if input.WebsiteRedirectLocation != "" { + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + } + input.prepareStorageClass(headers, isObs) + if input.Metadata != nil { + for key, value := range input.Metadata { + key = strings.TrimSpace(key) + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) + } + } + return +} + +func (input GetObjectInput) prepareResponseParams(params map[string]string) { + if input.ResponseCacheControl != "" { + params[PARAM_RESPONSE_CACHE_CONTROL] = input.ResponseCacheControl + } + if input.ResponseContentDisposition != "" { + params[PARAM_RESPONSE_CONTENT_DISPOSITION] = input.ResponseContentDisposition + } + if input.ResponseContentEncoding != "" { + params[PARAM_RESPONSE_CONTENT_ENCODING] = input.ResponseContentEncoding + } + if input.ResponseContentLanguage != "" { + params[PARAM_RESPONSE_CONTENT_LANGUAGE] = input.ResponseContentLanguage + } + if input.ResponseContentType != "" { + params[PARAM_RESPONSE_CONTENT_TYPE] = input.ResponseContentType + } + if input.ResponseExpires != "" { + params[PARAM_RESPONSE_EXPIRES] = input.ResponseExpires + } +} + +func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.GetObjectMetadataInput.trans(isObs) + if err != nil { + return + } + input.prepareResponseParams(params) + if input.ImageProcess != "" { + params[PARAM_IMAGE_PROCESS] = input.ImageProcess + } + if input.RangeStart >= 0 && input.RangeEnd > input.RangeStart { + headers[HEADER_RANGE] = []string{fmt.Sprintf("bytes=%d-%d", input.RangeStart, input.RangeEnd)} + } + + if input.IfMatch != "" { + headers[HEADER_IF_MATCH] = []string{input.IfMatch} + } + if input.IfNoneMatch != "" { + headers[HEADER_IF_NONE_MATCH] = []string{input.IfNoneMatch} + } + if !input.IfModifiedSince.IsZero() { + headers[HEADER_IF_MODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfModifiedSince)} + } + if !input.IfUnmodifiedSince.IsZero() { + headers[HEADER_IF_UNMODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfUnmodifiedSince)} + } + return +} + +func (input ObjectOperationInput) prepareGrantHeaders(headers map[string][]string) { + if GrantReadID := input.GrantReadId; GrantReadID != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{GrantReadID}, true) + } + if GrantReadAcpID := input.GrantReadAcpId; GrantReadAcpID != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{GrantReadAcpID}, true) + } + if GrantWriteAcpID := input.GrantWriteAcpId; GrantWriteAcpID != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{GrantWriteAcpID}, true) + } + if GrantFullControlID := input.GrantFullControlId; GrantFullControlID != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{GrantFullControlID}, true) + } +} + +func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + params = make(map[string]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } + input.prepareGrantHeaders(headers) + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) + } + if input.WebsiteRedirectLocation != "" { + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + + } + setSseHeader(headers, input.SseHeader, false, isObs) + if input.Expires != 0 { + setHeaders(headers, HEADER_EXPIRES, []string{Int64ToString(input.Expires)}, true) + } + if input.Metadata != nil { + for key, value := range input.Metadata { + key = strings.TrimSpace(key) + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) + } + } + return +} + +func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + + if input.ContentMD5 != "" { + headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} + } + + if input.ContentLength > 0 { + headers[HEADER_CONTENT_LENGTH_CAMEL] = []string{Int64ToString(input.ContentLength)} + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } + + return +} + +func (input PutObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.PutObjectBasicInput.trans(isObs) + if err != nil { + return + } + if input.Body != nil { + data = input.Body + } + return +} + +func (input AppendObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.PutObjectBasicInput.trans(isObs) + if err != nil { + return + } + params[string(SubResourceAppend)] = "" + params["position"] = strconv.FormatInt(input.Position, 10) + if input.Body != nil { + data = input.Body + } + return +} + +func (input ModifyObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string) + params = make(map[string]string) + params[string(SubResourceModify)] = "" + params["position"] = strconv.FormatInt(input.Position, 10) + if input.ContentLength > 0 { + headers[HEADER_CONTENT_LENGTH_CAMEL] = []string{Int64ToString(input.ContentLength)} + } + + if input.Body != nil { + data = input.Body + } + return +} + +func (input CopyObjectInput) prepareReplaceHeaders(headers map[string][]string) { + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL] = []string{input.CacheControl} + } + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE] = []string{input.ContentLanguage} + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE] = []string{input.ContentType} + } + if input.Expires != "" { + headers[HEADER_EXPIRES] = []string{input.Expires} + } +} + +func (input CopyObjectInput) prepareCopySourceHeaders(headers map[string][]string, isObs bool) { + if input.CopySourceIfMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MATCH, []string{input.CopySourceIfMatch}, isObs) + } + if input.CopySourceIfNoneMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_NONE_MATCH, []string{input.CopySourceIfNoneMatch}, isObs) + } + if !input.CopySourceIfModifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)}, isObs) + } + if !input.CopySourceIfUnmodifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)}, isObs) + } +} + +func (input CopyObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + + var copySource string + if input.CopySourceVersionId != "" { + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) + } else { + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) + } + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) + + if directive := string(input.MetadataDirective); directive != "" { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{directive}, isObs) + } + + if input.MetadataDirective == ReplaceMetadata { + input.prepareReplaceHeaders(headers) + } + + input.prepareCopySourceHeaders(headers, isObs) + if input.SourceSseHeader != nil { + if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } + } + if input.SuccessActionRedirect != "" { + headers[HEADER_SUCCESS_ACTION_REDIRECT] = []string{input.SuccessActionRedirect} + } + return +} + +func (input AbortMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId} + return +} + +func (input InitiateMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } + params[string(SubResourceUploads)] = "" + if input.EncodingType != "" { + params["encoding-type"] = input.EncodingType + } + return +} + +func (input UploadPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} + headers = make(map[string][]string) + setSseHeader(headers, input.SseHeader, true, isObs) + if input.ContentMD5 != "" { + headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} + } + if input.Body != nil { + data = input.Body + } + return +} + +func (input CompleteMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId} + if input.EncodingType != "" { + params["encoding-type"] = input.EncodingType + } + data, _ = ConvertCompleteMultipartUploadInputToXml(input, false) + return +} + +func (input ListPartsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId} + if input.MaxParts > 0 { + params["max-parts"] = IntToString(input.MaxParts) + } + if input.PartNumberMarker > 0 { + params["part-number-marker"] = IntToString(input.PartNumberMarker) + } + if input.EncodingType != "" { + params["encoding-type"] = input.EncodingType + } + return +} + +func (input CopyPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} + headers = make(map[string][]string, 1) + var copySource string + if input.CopySourceVersionId != "" { + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) + } else { + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) + } + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) + if input.CopySourceRangeStart >= 0 && input.CopySourceRangeEnd > input.CopySourceRangeStart { + setHeaders(headers, HEADER_COPY_SOURCE_RANGE, []string{fmt.Sprintf("bytes=%d-%d", input.CopySourceRangeStart, input.CopySourceRangeEnd)}, isObs) + } + + setSseHeader(headers, input.SseHeader, true, isObs) + if input.SourceSseHeader != nil { + if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } + + } + return +} + +func (input HeadObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input SetBucketRequestPaymentInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + return trans(SubResourceRequestPayment, input) +} + +type partSlice []Part + +func (parts partSlice) Len() int { + return len(parts) +} + +func (parts partSlice) Less(i, j int) bool { + return parts[i].PartNumber < parts[j].PartNumber +} + +func (parts partSlice) Swap(i, j int) { + parts[i], parts[j] = parts[j], parts[i] +} + +type readerWrapper struct { + reader io.Reader + mark int64 + totalCount int64 + readedCount int64 +} + +func (rw *readerWrapper) seek(offset int64, whence int) (int64, error) { + if r, ok := rw.reader.(*strings.Reader); ok { + return r.Seek(offset, whence) + } else if r, ok := rw.reader.(*bytes.Reader); ok { + return r.Seek(offset, whence) + } else if r, ok := rw.reader.(*os.File); ok { + return r.Seek(offset, whence) + } + return offset, nil +} + +func (rw *readerWrapper) Read(p []byte) (n int, err error) { + if rw.totalCount == 0 { + return 0, io.EOF + } + if rw.totalCount > 0 { + n, err = rw.reader.Read(p) + readedOnce := int64(n) + remainCount := rw.totalCount - rw.readedCount + if remainCount > readedOnce { + rw.readedCount += readedOnce + return n, err + } + rw.readedCount += remainCount + return int(remainCount), io.EOF + } + return rw.reader.Read(p) +} + +type fileReaderWrapper struct { + readerWrapper + filePath string +} + +func (input SetBucketFetchPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + contentType, _ := mimeTypes["json"] + headers = make(map[string][]string, 2) + headers[HEADER_CONTENT_TYPE] = []string{contentType} + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + data, err = convertFetchPolicyToJSON(input) + return +} + +func (input GetBucketFetchPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string, 1) + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + return +} + +func (input DeleteBucketFetchPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string, 1) + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + return +} + +func (input SetBucketFetchJobInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + contentType, _ := mimeTypes["json"] + headers = make(map[string][]string, 2) + headers[HEADER_CONTENT_TYPE] = []string{contentType} + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + data, err = convertFetchJobToJSON(input) + return +} + +func (input GetBucketFetchJobInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + headers = make(map[string][]string, 1) + setHeaders(headers, headerOefMarker, []string{"yes"}, isObs) + return +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/transfer.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/transfer.go new file mode 100644 index 0000000000..ce29e6e460 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/transfer.go @@ -0,0 +1,874 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "bufio" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sync" + "sync/atomic" + "syscall" +) + +var errAbort = errors.New("AbortError") + +// FileStatus defines the upload file properties +type FileStatus struct { + XMLName xml.Name `xml:"FileInfo"` + LastModified int64 `xml:"LastModified"` + Size int64 `xml:"Size"` +} + +// UploadPartInfo defines the upload part properties +type UploadPartInfo struct { + XMLName xml.Name `xml:"UploadPart"` + PartNumber int `xml:"PartNumber"` + Etag string `xml:"Etag"` + PartSize int64 `xml:"PartSize"` + Offset int64 `xml:"Offset"` + IsCompleted bool `xml:"IsCompleted"` +} + +// UploadCheckpoint defines the upload checkpoint file properties +type UploadCheckpoint struct { + XMLName xml.Name `xml:"UploadFileCheckpoint"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId,omitempty"` + UploadFile string `xml:"FileUrl"` + FileInfo FileStatus `xml:"FileInfo"` + UploadParts []UploadPartInfo `xml:"UploadParts>UploadPart"` +} + +func (ufc *UploadCheckpoint) isValid(bucket, key, uploadFile string, fileStat os.FileInfo) bool { + if ufc.Bucket != bucket || ufc.Key != key || ufc.UploadFile != uploadFile { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the bucketName or objectKey or uploadFile was changed. clear the record.") + return false + } + + if ufc.FileInfo.Size != fileStat.Size() || ufc.FileInfo.LastModified != fileStat.ModTime().Unix() { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the uploadFile was changed. clear the record.") + return false + } + + if ufc.UploadId == "" { + doLog(LEVEL_INFO, "UploadId is invalid. clear the record.") + return false + } + + return true +} + +type uploadPartTask struct { + UploadPartInput + obsClient *ObsClient + abort *int32 + extensions []extensionOptions + enableCheckpoint bool +} + +func (task *uploadPartTask) Run() interface{} { + if atomic.LoadInt32(task.abort) == 1 { + return errAbort + } + + input := &UploadPartInput{} + input.Bucket = task.Bucket + input.Key = task.Key + input.PartNumber = task.PartNumber + input.UploadId = task.UploadId + input.SseHeader = task.SseHeader + input.SourceFile = task.SourceFile + input.Offset = task.Offset + input.PartSize = task.PartSize + extensions := task.extensions + + var output *UploadPartOutput + var err error + if extensions != nil { + output, err = task.obsClient.UploadPart(input, extensions...) + } else { + output, err = task.obsClient.UploadPart(input) + } + + if err == nil { + if output.ETag == "" { + doLog(LEVEL_WARN, "Get invalid etag value after uploading part [%d].", task.PartNumber) + if !task.enableCheckpoint { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.PartNumber) + } + return fmt.Errorf("get invalid etag value after uploading part [%d]", task.PartNumber) + } + return output + } else if obsError, ok := err.(ObsError); ok && obsError.StatusCode >= 400 && obsError.StatusCode < 500 { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.PartNumber) + } + return err +} + +func loadCheckpointFile(checkpointFile string, result interface{}) error { + ret, err := ioutil.ReadFile(checkpointFile) + if err != nil { + return err + } + if len(ret) == 0 { + return nil + } + return xml.Unmarshal(ret, result) +} + +func updateCheckpointFile(fc interface{}, checkpointFilePath string) error { + result, err := xml.Marshal(fc) + if err != nil { + return err + } + err = ioutil.WriteFile(checkpointFilePath, result, 0666) + return err +} + +func getCheckpointFile(ufc *UploadCheckpoint, uploadFileStat os.FileInfo, input *UploadFileInput, obsClient *ObsClient, extensions []extensionOptions) (needCheckpoint bool, err error) { + checkpointFilePath := input.CheckpointFile + checkpointFileStat, err := os.Stat(checkpointFilePath) + if err != nil { + doLog(LEVEL_DEBUG, fmt.Sprintf("Stat checkpoint file failed with error: [%v].", err)) + return true, nil + } + if checkpointFileStat.IsDir() { + doLog(LEVEL_ERROR, "Checkpoint file can not be a folder.") + return false, errors.New("checkpoint file can not be a folder") + } + err = loadCheckpointFile(checkpointFilePath, ufc) + if err != nil { + doLog(LEVEL_WARN, fmt.Sprintf("Load checkpoint file failed with error: [%v].", err)) + return true, nil + } else if !ufc.isValid(input.Bucket, input.Key, input.UploadFile, uploadFileStat) { + if ufc.Bucket != "" && ufc.Key != "" && ufc.UploadId != "" { + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort upload task [%s].", ufc.UploadId) + } + } + _err := os.Remove(checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, fmt.Sprintf("Failed to remove checkpoint file with error: [%v].", _err)) + } + } else { + return false, nil + } + + return true, nil +} + +func prepareUpload(ufc *UploadCheckpoint, uploadFileStat os.FileInfo, input *UploadFileInput, obsClient *ObsClient, extensions []extensionOptions) error { + initiateInput := &InitiateMultipartUploadInput{} + initiateInput.ObjectOperationInput = input.ObjectOperationInput + initiateInput.ContentType = input.ContentType + initiateInput.EncodingType = input.EncodingType + var output *InitiateMultipartUploadOutput + var err error + if extensions != nil { + output, err = obsClient.InitiateMultipartUpload(initiateInput, extensions...) + } else { + output, err = obsClient.InitiateMultipartUpload(initiateInput) + } + if err != nil { + return err + } + + ufc.Bucket = input.Bucket + ufc.Key = input.Key + ufc.UploadFile = input.UploadFile + ufc.FileInfo = FileStatus{} + ufc.FileInfo.Size = uploadFileStat.Size() + ufc.FileInfo.LastModified = uploadFileStat.ModTime().Unix() + ufc.UploadId = output.UploadId + + err = sliceFile(input.PartSize, ufc) + return err +} + +func sliceFile(partSize int64, ufc *UploadCheckpoint) error { + fileSize := ufc.FileInfo.Size + cnt := fileSize / partSize + if cnt >= 10000 { + partSize = fileSize / 10000 + if fileSize%10000 != 0 { + partSize++ + } + cnt = fileSize / partSize + } + if fileSize%partSize != 0 { + cnt++ + } + + if partSize > MAX_PART_SIZE { + doLog(LEVEL_ERROR, "The source upload file is too large") + return fmt.Errorf("The source upload file is too large") + } + + if cnt == 0 { + uploadPart := UploadPartInfo{} + uploadPart.PartNumber = 1 + ufc.UploadParts = []UploadPartInfo{uploadPart} + } else { + uploadParts := make([]UploadPartInfo, 0, cnt) + var i int64 + for i = 0; i < cnt; i++ { + uploadPart := UploadPartInfo{} + uploadPart.PartNumber = int(i) + 1 + uploadPart.PartSize = partSize + uploadPart.Offset = i * partSize + uploadParts = append(uploadParts, uploadPart) + } + if value := fileSize % partSize; value != 0 { + uploadParts[cnt-1].PartSize = value + } + ufc.UploadParts = uploadParts + } + return nil +} + +func abortTask(bucket, key, uploadID string, obsClient *ObsClient, extensions []extensionOptions) error { + input := &AbortMultipartUploadInput{} + input.Bucket = bucket + input.Key = key + input.UploadId = uploadID + if extensions != nil { + _, err := obsClient.AbortMultipartUpload(input, extensions...) + return err + } + _, err := obsClient.AbortMultipartUpload(input) + return err +} + +func handleUploadFileResult(uploadPartError error, ufc *UploadCheckpoint, enableCheckpoint bool, obsClient *ObsClient, extensions []extensionOptions) error { + if uploadPartError != nil { + if enableCheckpoint { + return uploadPartError + } + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort task [%s].", ufc.UploadId) + } + return uploadPartError + } + return nil +} + +func completeParts(ufc *UploadCheckpoint, enableCheckpoint bool, checkpointFilePath string, obsClient *ObsClient, encodingType string, extensions []extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + completeInput := &CompleteMultipartUploadInput{} + completeInput.Bucket = ufc.Bucket + completeInput.Key = ufc.Key + completeInput.UploadId = ufc.UploadId + completeInput.EncodingType = encodingType + parts := make([]Part, 0, len(ufc.UploadParts)) + for _, uploadPart := range ufc.UploadParts { + part := Part{} + part.PartNumber = uploadPart.PartNumber + part.ETag = uploadPart.Etag + parts = append(parts, part) + } + completeInput.Parts = parts + var completeOutput *CompleteMultipartUploadOutput + if extensions != nil { + completeOutput, err = obsClient.CompleteMultipartUpload(completeInput, extensions...) + } else { + completeOutput, err = obsClient.CompleteMultipartUpload(completeInput) + } + + if err == nil { + if enableCheckpoint { + _err := os.Remove(checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, "Upload file successfully, but remove checkpoint file failed with error [%v].", _err) + } + } + return completeOutput, err + } + if !enableCheckpoint { + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort task [%s].", ufc.UploadId) + } + } + return completeOutput, err +} + +func (obsClient ObsClient) resumeUpload(input *UploadFileInput, extensions []extensionOptions) (output *CompleteMultipartUploadOutput, err error) { + uploadFileStat, err := os.Stat(input.UploadFile) + if err != nil { + doLog(LEVEL_ERROR, fmt.Sprintf("Failed to stat uploadFile with error: [%v].", err)) + return nil, err + } + if uploadFileStat.IsDir() { + doLog(LEVEL_ERROR, "UploadFile can not be a folder.") + return nil, errors.New("uploadFile can not be a folder") + } + + ufc := &UploadCheckpoint{} + + var needCheckpoint = true + var checkpointFilePath = input.CheckpointFile + var enableCheckpoint = input.EnableCheckpoint + if enableCheckpoint { + needCheckpoint, err = getCheckpointFile(ufc, uploadFileStat, input, &obsClient, extensions) + if err != nil { + return nil, err + } + } + if needCheckpoint { + err = prepareUpload(ufc, uploadFileStat, input, &obsClient, extensions) + if err != nil { + return nil, err + } + + if enableCheckpoint { + err = updateCheckpointFile(ufc, checkpointFilePath) + if err != nil { + doLog(LEVEL_ERROR, "Failed to update checkpoint file with error [%v].", err) + _err := abortTask(ufc.Bucket, ufc.Key, ufc.UploadId, &obsClient, extensions) + if _err != nil { + doLog(LEVEL_WARN, "Failed to abort task [%s].", ufc.UploadId) + } + return nil, err + } + } + } + + uploadPartError := obsClient.uploadPartConcurrent(ufc, checkpointFilePath, input, extensions) + err = handleUploadFileResult(uploadPartError, ufc, enableCheckpoint, &obsClient, extensions) + if err != nil { + return nil, err + } + + completeOutput, err := completeParts(ufc, enableCheckpoint, checkpointFilePath, &obsClient, input.EncodingType, extensions) + + return completeOutput, err +} + +func handleUploadTaskResult(result interface{}, ufc *UploadCheckpoint, partNum int, enableCheckpoint bool, checkpointFilePath string, lock *sync.Mutex) (err error) { + if uploadPartOutput, ok := result.(*UploadPartOutput); ok { + lock.Lock() + defer lock.Unlock() + ufc.UploadParts[partNum-1].Etag = uploadPartOutput.ETag + ufc.UploadParts[partNum-1].IsCompleted = true + if enableCheckpoint { + _err := updateCheckpointFile(ufc, checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, "Failed to update checkpoint file with error [%v].", _err) + } + } + } else if result != errAbort { + if _err, ok := result.(error); ok { + err = _err + } + } + return +} + +func (obsClient ObsClient) uploadPartConcurrent(ufc *UploadCheckpoint, checkpointFilePath string, input *UploadFileInput, extensions []extensionOptions) error { + pool := NewRoutinePool(input.TaskNum, MAX_PART_NUM) + var uploadPartError atomic.Value + var errFlag int32 + var abort int32 + lock := new(sync.Mutex) + for _, uploadPart := range ufc.UploadParts { + if atomic.LoadInt32(&abort) == 1 { + break + } + if uploadPart.IsCompleted { + continue + } + task := uploadPartTask{ + UploadPartInput: UploadPartInput{ + Bucket: ufc.Bucket, + Key: ufc.Key, + PartNumber: uploadPart.PartNumber, + UploadId: ufc.UploadId, + SseHeader: input.SseHeader, + SourceFile: input.UploadFile, + Offset: uploadPart.Offset, + PartSize: uploadPart.PartSize, + }, + obsClient: &obsClient, + abort: &abort, + extensions: extensions, + enableCheckpoint: input.EnableCheckpoint, + } + pool.ExecuteFunc(func() interface{} { + result := task.Run() + err := handleUploadTaskResult(result, ufc, task.PartNumber, input.EnableCheckpoint, input.CheckpointFile, lock) + if err != nil && atomic.CompareAndSwapInt32(&errFlag, 0, 1) { + uploadPartError.Store(err) + } + return nil + }) + } + pool.ShutDown() + if err, ok := uploadPartError.Load().(error); ok { + return err + } + return nil +} + +// ObjectInfo defines download object info +type ObjectInfo struct { + XMLName xml.Name `xml:"ObjectInfo"` + LastModified int64 `xml:"LastModified"` + Size int64 `xml:"Size"` + ETag string `xml:"ETag"` +} + +// TempFileInfo defines temp download file properties +type TempFileInfo struct { + XMLName xml.Name `xml:"TempFileInfo"` + TempFileUrl string `xml:"TempFileUrl"` + Size int64 `xml:"Size"` +} + +// DownloadPartInfo defines download part properties +type DownloadPartInfo struct { + XMLName xml.Name `xml:"DownloadPart"` + PartNumber int64 `xml:"PartNumber"` + RangeEnd int64 `xml:"RangeEnd"` + Offset int64 `xml:"Offset"` + IsCompleted bool `xml:"IsCompleted"` +} + +// DownloadCheckpoint defines download checkpoint file properties +type DownloadCheckpoint struct { + XMLName xml.Name `xml:"DownloadFileCheckpoint"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` + DownloadFile string `xml:"FileUrl"` + ObjectInfo ObjectInfo `xml:"ObjectInfo"` + TempFileInfo TempFileInfo `xml:"TempFileInfo"` + DownloadParts []DownloadPartInfo `xml:"DownloadParts>DownloadPart"` +} + +func (dfc *DownloadCheckpoint) isValid(input *DownloadFileInput, output *GetObjectMetadataOutput) bool { + if dfc.Bucket != input.Bucket || dfc.Key != input.Key || dfc.VersionId != input.VersionId || dfc.DownloadFile != input.DownloadFile { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the bucketName or objectKey or downloadFile was changed. clear the record.") + return false + } + if dfc.ObjectInfo.LastModified != output.LastModified.Unix() || dfc.ObjectInfo.ETag != output.ETag || dfc.ObjectInfo.Size != output.ContentLength { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the object info was changed. clear the record.") + return false + } + if dfc.TempFileInfo.Size != output.ContentLength { + doLog(LEVEL_INFO, "Checkpoint file is invalid, size was changed. clear the record.") + return false + } + stat, err := os.Stat(dfc.TempFileInfo.TempFileUrl) + if err != nil || stat.Size() != dfc.ObjectInfo.Size { + doLog(LEVEL_INFO, "Checkpoint file is invalid, the temp download file was changed. clear the record.") + return false + } + + return true +} + +type downloadPartTask struct { + GetObjectInput + obsClient *ObsClient + extensions []extensionOptions + abort *int32 + partNumber int64 + tempFileURL string + enableCheckpoint bool +} + +func (task *downloadPartTask) Run() interface{} { + if atomic.LoadInt32(task.abort) == 1 { + return errAbort + } + getObjectInput := &GetObjectInput{} + getObjectInput.GetObjectMetadataInput = task.GetObjectMetadataInput + getObjectInput.IfMatch = task.IfMatch + getObjectInput.IfNoneMatch = task.IfNoneMatch + getObjectInput.IfModifiedSince = task.IfModifiedSince + getObjectInput.IfUnmodifiedSince = task.IfUnmodifiedSince + getObjectInput.RangeStart = task.RangeStart + getObjectInput.RangeEnd = task.RangeEnd + + var output *GetObjectOutput + var err error + if task.extensions != nil { + output, err = task.obsClient.GetObject(getObjectInput, task.extensions...) + } else { + output, err = task.obsClient.GetObject(getObjectInput) + } + + if err == nil { + defer func() { + errMsg := output.Body.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close response body.") + } + }() + _err := updateDownloadFile(task.tempFileURL, task.RangeStart, output) + if _err != nil { + if !task.enableCheckpoint { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.partNumber) + } + return _err + } + return output + } else if obsError, ok := err.(ObsError); ok && obsError.StatusCode >= 400 && obsError.StatusCode < 500 { + atomic.CompareAndSwapInt32(task.abort, 0, 1) + doLog(LEVEL_WARN, "Task is aborted, part number is [%d]", task.partNumber) + } + return err +} + +func getObjectInfo(input *DownloadFileInput, obsClient *ObsClient, extensions []extensionOptions) (getObjectmetaOutput *GetObjectMetadataOutput, err error) { + if extensions != nil { + getObjectmetaOutput, err = obsClient.GetObjectMetadata(&input.GetObjectMetadataInput, extensions...) + } else { + getObjectmetaOutput, err = obsClient.GetObjectMetadata(&input.GetObjectMetadataInput) + } + + return +} + +func getDownloadCheckpointFile(dfc *DownloadCheckpoint, input *DownloadFileInput, output *GetObjectMetadataOutput) (needCheckpoint bool, err error) { + checkpointFilePath := input.CheckpointFile + checkpointFileStat, err := os.Stat(checkpointFilePath) + if err != nil { + doLog(LEVEL_DEBUG, fmt.Sprintf("Stat checkpoint file failed with error: [%v].", err)) + return true, nil + } + if checkpointFileStat.IsDir() { + doLog(LEVEL_ERROR, "Checkpoint file can not be a folder.") + return false, errors.New("checkpoint file can not be a folder") + } + err = loadCheckpointFile(checkpointFilePath, dfc) + if err != nil { + doLog(LEVEL_WARN, fmt.Sprintf("Load checkpoint file failed with error: [%v].", err)) + return true, nil + } else if !dfc.isValid(input, output) { + if dfc.TempFileInfo.TempFileUrl != "" { + _err := os.Remove(dfc.TempFileInfo.TempFileUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to remove temp download file with error [%v].", _err) + } + } + _err := os.Remove(checkpointFilePath) + if _err != nil { + doLog(LEVEL_WARN, "Failed to remove checkpoint file with error [%v].", _err) + } + } else { + return false, nil + } + + return true, nil +} + +func sliceObject(objectSize, partSize int64, dfc *DownloadCheckpoint) { + cnt := objectSize / partSize + if objectSize%partSize > 0 { + cnt++ + } + + if cnt == 0 { + downloadPart := DownloadPartInfo{} + downloadPart.PartNumber = 1 + dfc.DownloadParts = []DownloadPartInfo{downloadPart} + } else { + downloadParts := make([]DownloadPartInfo, 0, cnt) + var i int64 + for i = 0; i < cnt; i++ { + downloadPart := DownloadPartInfo{} + downloadPart.PartNumber = i + 1 + downloadPart.Offset = i * partSize + downloadPart.RangeEnd = (i+1)*partSize - 1 + downloadParts = append(downloadParts, downloadPart) + } + dfc.DownloadParts = downloadParts + if value := objectSize % partSize; value > 0 { + dfc.DownloadParts[cnt-1].RangeEnd = dfc.ObjectInfo.Size - 1 + } + } +} + +func createFile(tempFileURL string, fileSize int64) error { + fd, err := syscall.Open(tempFileURL, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + doLog(LEVEL_WARN, "Failed to open temp download file [%s].", tempFileURL) + return err + } + defer func() { + errMsg := syscall.Close(fd) + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with error [%v].", errMsg) + } + }() + err = syscall.Ftruncate(fd, fileSize) + if err != nil { + doLog(LEVEL_WARN, "Failed to create file with error [%v].", err) + } + return err +} + +func prepareTempFile(tempFileURL string, fileSize int64) error { + parentDir := filepath.Dir(tempFileURL) + stat, err := os.Stat(parentDir) + if err != nil { + doLog(LEVEL_DEBUG, "Failed to stat path with error [%v].", err) + _err := os.MkdirAll(parentDir, os.ModePerm) + if _err != nil { + doLog(LEVEL_ERROR, "Failed to make dir with error [%v].", _err) + return _err + } + } else if !stat.IsDir() { + doLog(LEVEL_ERROR, "Cannot create folder [%s] due to a same file exists.", parentDir) + return fmt.Errorf("cannot create folder [%s] due to a same file exists", parentDir) + } + + err = createFile(tempFileURL, fileSize) + if err == nil { + return nil + } + fd, err := os.OpenFile(tempFileURL, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + doLog(LEVEL_ERROR, "Failed to open temp download file [%s].", tempFileURL) + return err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with error [%v].", errMsg) + } + }() + if fileSize > 0 { + _, err = fd.WriteAt([]byte("a"), fileSize-1) + if err != nil { + doLog(LEVEL_ERROR, "Failed to create temp download file with error [%v].", err) + return err + } + } + + return nil +} + +func handleDownloadFileResult(tempFileURL string, enableCheckpoint bool, downloadFileError error) error { + if downloadFileError != nil { + if !enableCheckpoint { + _err := os.Remove(tempFileURL) + if _err != nil { + doLog(LEVEL_WARN, "Failed to remove temp download file with error [%v].", _err) + } + } + return downloadFileError + } + return nil +} + +func (obsClient ObsClient) resumeDownload(input *DownloadFileInput, extensions []extensionOptions) (output *GetObjectMetadataOutput, err error) { + getObjectmetaOutput, err := getObjectInfo(input, &obsClient, extensions) + if err != nil { + return nil, err + } + + objectSize := getObjectmetaOutput.ContentLength + partSize := input.PartSize + dfc := &DownloadCheckpoint{} + + var needCheckpoint = true + var checkpointFilePath = input.CheckpointFile + var enableCheckpoint = input.EnableCheckpoint + if enableCheckpoint { + needCheckpoint, err = getDownloadCheckpointFile(dfc, input, getObjectmetaOutput) + if err != nil { + return nil, err + } + } + + if needCheckpoint { + dfc.Bucket = input.Bucket + dfc.Key = input.Key + dfc.VersionId = input.VersionId + dfc.DownloadFile = input.DownloadFile + dfc.ObjectInfo = ObjectInfo{} + dfc.ObjectInfo.LastModified = getObjectmetaOutput.LastModified.Unix() + dfc.ObjectInfo.Size = getObjectmetaOutput.ContentLength + dfc.ObjectInfo.ETag = getObjectmetaOutput.ETag + dfc.TempFileInfo = TempFileInfo{} + dfc.TempFileInfo.TempFileUrl = input.DownloadFile + ".tmp" + dfc.TempFileInfo.Size = getObjectmetaOutput.ContentLength + + sliceObject(objectSize, partSize, dfc) + _err := prepareTempFile(dfc.TempFileInfo.TempFileUrl, dfc.TempFileInfo.Size) + if _err != nil { + return nil, _err + } + + if enableCheckpoint { + _err := updateCheckpointFile(dfc, checkpointFilePath) + if _err != nil { + doLog(LEVEL_ERROR, "Failed to update checkpoint file with error [%v].", _err) + _errMsg := os.Remove(dfc.TempFileInfo.TempFileUrl) + if _errMsg != nil { + doLog(LEVEL_WARN, "Failed to remove temp download file with error [%v].", _errMsg) + } + return nil, _err + } + } + } + + downloadFileError := obsClient.downloadFileConcurrent(input, dfc, extensions) + err = handleDownloadFileResult(dfc.TempFileInfo.TempFileUrl, enableCheckpoint, downloadFileError) + if err != nil { + return nil, err + } + + err = os.Rename(dfc.TempFileInfo.TempFileUrl, input.DownloadFile) + if err != nil { + doLog(LEVEL_ERROR, "Failed to rename temp download file [%s] to download file [%s] with error [%v].", dfc.TempFileInfo.TempFileUrl, input.DownloadFile, err) + return nil, err + } + if enableCheckpoint { + err = os.Remove(checkpointFilePath) + if err != nil { + doLog(LEVEL_WARN, "Download file successfully, but remove checkpoint file failed with error [%v].", err) + } + } + + return getObjectmetaOutput, nil +} + +func updateDownloadFile(filePath string, rangeStart int64, output *GetObjectOutput) error { + fd, err := os.OpenFile(filePath, os.O_WRONLY, 0666) + if err != nil { + doLog(LEVEL_ERROR, "Failed to open file [%s].", filePath) + return err + } + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with error [%v].", errMsg) + } + }() + _, err = fd.Seek(rangeStart, 0) + if err != nil { + doLog(LEVEL_ERROR, "Failed to seek file with error [%v].", err) + return err + } + fileWriter := bufio.NewWriterSize(fd, 65536) + part := make([]byte, 8192) + var readErr error + var readCount int + for { + readCount, readErr = output.Body.Read(part) + if readCount > 0 { + wcnt, werr := fileWriter.Write(part[0:readCount]) + if werr != nil { + doLog(LEVEL_ERROR, "Failed to write to file with error [%v].", werr) + return werr + } + if wcnt != readCount { + doLog(LEVEL_ERROR, "Failed to write to file [%s], expect: [%d], actual: [%d]", filePath, readCount, wcnt) + return fmt.Errorf("Failed to write to file [%s], expect: [%d], actual: [%d]", filePath, readCount, wcnt) + } + } + if readErr != nil { + if readErr != io.EOF { + doLog(LEVEL_ERROR, "Failed to read response body with error [%v].", readErr) + return readErr + } + break + } + } + err = fileWriter.Flush() + if err != nil { + doLog(LEVEL_ERROR, "Failed to flush file with error [%v].", err) + return err + } + return nil +} + +func handleDownloadTaskResult(result interface{}, dfc *DownloadCheckpoint, partNum int64, enableCheckpoint bool, checkpointFile string, lock *sync.Mutex) (err error) { + if _, ok := result.(*GetObjectOutput); ok { + lock.Lock() + defer lock.Unlock() + dfc.DownloadParts[partNum-1].IsCompleted = true + if enableCheckpoint { + _err := updateCheckpointFile(dfc, checkpointFile) + if _err != nil { + doLog(LEVEL_WARN, "Failed to update checkpoint file with error [%v].", _err) + } + } + } else if result != errAbort { + if _err, ok := result.(error); ok { + err = _err + } + } + return +} + +func (obsClient ObsClient) downloadFileConcurrent(input *DownloadFileInput, dfc *DownloadCheckpoint, extensions []extensionOptions) error { + pool := NewRoutinePool(input.TaskNum, MAX_PART_NUM) + var downloadPartError atomic.Value + var errFlag int32 + var abort int32 + lock := new(sync.Mutex) + for _, downloadPart := range dfc.DownloadParts { + if atomic.LoadInt32(&abort) == 1 { + break + } + if downloadPart.IsCompleted { + continue + } + task := downloadPartTask{ + GetObjectInput: GetObjectInput{ + GetObjectMetadataInput: input.GetObjectMetadataInput, + IfMatch: input.IfMatch, + IfNoneMatch: input.IfNoneMatch, + IfUnmodifiedSince: input.IfUnmodifiedSince, + IfModifiedSince: input.IfModifiedSince, + RangeStart: downloadPart.Offset, + RangeEnd: downloadPart.RangeEnd, + }, + obsClient: &obsClient, + extensions: extensions, + abort: &abort, + partNumber: downloadPart.PartNumber, + tempFileURL: dfc.TempFileInfo.TempFileUrl, + enableCheckpoint: input.EnableCheckpoint, + } + pool.ExecuteFunc(func() interface{} { + result := task.Run() + err := handleDownloadTaskResult(result, dfc, task.partNumber, input.EnableCheckpoint, input.CheckpointFile, lock) + if err != nil && atomic.CompareAndSwapInt32(&errFlag, 0, 1) { + downloadPartError.Store(err) + } + return nil + }) + } + pool.ShutDown() + if err, ok := downloadPartError.Load().(error); ok { + return err + } + + return nil +} diff --git a/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/util.go b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/util.go new file mode 100644 index 0000000000..6ee53826c6 --- /dev/null +++ b/vendor/github.com/huaweicloud/huaweicloud-sdk-go-obs/obs/util.go @@ -0,0 +1,551 @@ +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package obs + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/xml" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +var regex = regexp.MustCompile("^[\u4e00-\u9fa5]$") +var ipRegex = regexp.MustCompile("^((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)$") +var v4AuthRegex = regexp.MustCompile("Credential=(.+?),SignedHeaders=(.+?),Signature=.+") +var regionRegex = regexp.MustCompile(".+/\\d+/(.+?)/.+") + +// StringContains replaces subStr in src with subTranscoding and returns the new string +func StringContains(src string, subStr string, subTranscoding string) string { + return strings.Replace(src, subStr, subTranscoding, -1) +} + +// XmlTranscoding replaces special characters with their escaped form +func XmlTranscoding(src string) string { + srcTmp := StringContains(src, "&", "&") + srcTmp = StringContains(srcTmp, "<", "<") + srcTmp = StringContains(srcTmp, ">", ">") + srcTmp = StringContains(srcTmp, "'", "'") + srcTmp = StringContains(srcTmp, "\"", """) + return srcTmp +} + +// StringToInt converts string value to int value with default value +func StringToInt(value string, def int) int { + ret, err := strconv.Atoi(value) + if err != nil { + ret = def + } + return ret +} + +// StringToInt64 converts string value to int64 value with default value +func StringToInt64(value string, def int64) int64 { + ret, err := strconv.ParseInt(value, 10, 64) + if err != nil { + ret = def + } + return ret +} + +// IntToString converts int value to string value +func IntToString(value int) string { + return strconv.Itoa(value) +} + +// Int64ToString converts int64 value to string value +func Int64ToString(value int64) string { + return strconv.FormatInt(value, 10) +} + +// GetCurrentTimestamp gets unix time in milliseconds +func GetCurrentTimestamp() int64 { + return time.Now().UnixNano() / 1000000 +} + +// FormatUtcNow gets a textual representation of the UTC format time value +func FormatUtcNow(format string) string { + return time.Now().UTC().Format(format) +} + +// FormatUtcToRfc1123 gets a textual representation of the RFC1123 format time value +func FormatUtcToRfc1123(t time.Time) string { + ret := t.UTC().Format(time.RFC1123) + return ret[:strings.LastIndex(ret, "UTC")] + "GMT" +} + +// Md5 gets the md5 value of input +func Md5(value []byte) []byte { + m := md5.New() + _, err := m.Write(value) + if err != nil { + doLog(LEVEL_WARN, "MD5 failed to write") + } + return m.Sum(nil) +} + +// HmacSha1 gets hmac sha1 value of input +func HmacSha1(key, value []byte) []byte { + mac := hmac.New(sha1.New, key) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha1 failed to write") + } + return mac.Sum(nil) +} + +// HmacSha256 get hmac sha256 value if input +func HmacSha256(key, value []byte) []byte { + mac := hmac.New(sha256.New, key) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha256 failed to write") + } + return mac.Sum(nil) +} + +// Base64Encode wrapper of base64.StdEncoding.EncodeToString +func Base64Encode(value []byte) string { + return base64.StdEncoding.EncodeToString(value) +} + +// Base64Decode wrapper of base64.StdEncoding.DecodeString +func Base64Decode(value string) ([]byte, error) { + return base64.StdEncoding.DecodeString(value) +} + +// HexMd5 returns the md5 value of input in hexadecimal format +func HexMd5(value []byte) string { + return Hex(Md5(value)) +} + +// Base64Md5 returns the md5 value of input with Base64Encode +func Base64Md5(value []byte) string { + return Base64Encode(Md5(value)) +} + +// Sha256Hash returns sha256 checksum +func Sha256Hash(value []byte) []byte { + hash := sha256.New() + _, err := hash.Write(value) + if err != nil { + doLog(LEVEL_WARN, "Sha256Hash failed to write") + } + return hash.Sum(nil) +} + +// ParseXml wrapper of xml.Unmarshal +func ParseXml(value []byte, result interface{}) error { + if len(value) == 0 { + return nil + } + return xml.Unmarshal(value, result) +} + +// parseJSON wrapper of json.Unmarshal +func parseJSON(value []byte, result interface{}) error { + if len(value) == 0 { + return nil + } + return json.Unmarshal(value, result) +} + +// TransToXml wrapper of xml.Marshal +func TransToXml(value interface{}) ([]byte, error) { + if value == nil { + return []byte{}, nil + } + return xml.Marshal(value) +} + +// Hex wrapper of hex.EncodeToString +func Hex(value []byte) string { + return hex.EncodeToString(value) +} + +// HexSha256 returns the Sha256Hash value of input in hexadecimal format +func HexSha256(value []byte) string { + return Hex(Sha256Hash(value)) +} + +// UrlDecode wrapper of url.QueryUnescape +func UrlDecode(value string) (string, error) { + ret, err := url.QueryUnescape(value) + if err == nil { + return ret, nil + } + return "", err +} + +// UrlDecodeWithoutError wrapper of UrlDecode +func UrlDecodeWithoutError(value string) string { + ret, err := UrlDecode(value) + if err == nil { + return ret + } + if isErrorLogEnabled() { + doLog(LEVEL_ERROR, "Url decode error") + } + return "" +} + +// IsIP checks whether the value matches ip address +func IsIP(value string) bool { + return ipRegex.MatchString(value) +} + +// UrlEncode encodes the input value +func UrlEncode(value string, chineseOnly bool) string { + if chineseOnly { + values := make([]string, 0, len(value)) + for _, val := range value { + _value := string(val) + if regex.MatchString(_value) { + _value = url.QueryEscape(_value) + } + values = append(values, _value) + } + return strings.Join(values, "") + } + return url.QueryEscape(value) +} + +func copyHeaders(m map[string][]string) (ret map[string][]string) { + if m != nil { + ret = make(map[string][]string, len(m)) + for key, values := range m { + _values := make([]string, 0, len(values)) + for _, value := range values { + _values = append(_values, value) + } + ret[strings.ToLower(key)] = _values + } + } else { + ret = make(map[string][]string) + } + + return +} + +func parseHeaders(headers map[string][]string) (signature string, region string, signedHeaders string) { + signature = "v2" + if receviedAuthorization, ok := headers[strings.ToLower(HEADER_AUTH_CAMEL)]; ok && len(receviedAuthorization) > 0 { + if strings.HasPrefix(receviedAuthorization[0], V4_HASH_PREFIX) { + signature = "v4" + matches := v4AuthRegex.FindStringSubmatch(receviedAuthorization[0]) + if len(matches) >= 3 { + region = matches[1] + regions := regionRegex.FindStringSubmatch(region) + if len(regions) >= 2 { + region = regions[1] + } + signedHeaders = matches[2] + } + + } else if strings.HasPrefix(receviedAuthorization[0], V2_HASH_PREFIX) { + signature = "v2" + } + } + return +} + +func getTemporaryKeys() []string { + return []string{ + "Signature", + "signature", + "X-Amz-Signature", + "x-amz-signature", + } +} + +func getIsObs(isTemporary bool, querys []string, headers map[string][]string) bool { + isObs := true + if isTemporary { + for _, value := range querys { + keyPrefix := strings.ToLower(value) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + } else if strings.HasPrefix(value, HEADER_ACCESSS_KEY_AMZ) { + isObs = false + } + } + } else { + for key := range headers { + keyPrefix := strings.ToLower(key) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + break + } + } + } + return isObs +} + +func isPathStyle(headers map[string][]string, bucketName string) bool { + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + return true + } + return false +} + +// GetV2Authorization v2 Authorization +func GetV2Authorization(ak, sk, method, bucketName, objectKey, queryURL string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryURL, "?") { + queryURL = queryURL[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryURL, "&") + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + params := make(map[string]string) + + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key := UrlDecodeWithoutError(kv[0]) + params[key] = "" + } else if length >= 2 { + key := UrlDecodeWithoutError(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val := UrlDecodeWithoutError(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + headers = copyHeaders(headers) + pathStyle := isPathStyle(headers, bucketName) + conf := &config{securityProviders: []securityProvider{NewBasicSecurityProvider(ak, sk, "")}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + conf.signature = SignatureObs + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedURL, headers, true) + v2HashPrefix := OBS_HASH_PREFIX + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + return +} + +func getQuerysResult(querys []string) []string { + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + return querysResult +} + +func getParams(querysResult []string) map[string]string { + params := make(map[string]string) + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key := UrlDecodeWithoutError(kv[0]) + params[key] = "" + } else if length >= 2 { + key := UrlDecodeWithoutError(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val := UrlDecodeWithoutError(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + return params +} + +func getTemporaryAndSignature(params map[string]string) (bool, string) { + isTemporary := false + signature := "v2" + temporaryKeys := getTemporaryKeys() + for _, key := range temporaryKeys { + if _, ok := params[key]; ok { + isTemporary = true + if strings.ToLower(key) == "signature" { + signature = "v2" + } else if strings.ToLower(key) == "x-amz-signature" { + signature = "v4" + } + break + } + } + return isTemporary, signature +} + +// GetAuthorization Authorization +func GetAuthorization(ak, sk, method, bucketName, objectKey, queryURL string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryURL, "?") { + queryURL = queryURL[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryURL, "&") + querysResult := getQuerysResult(querys) + params := getParams(querysResult) + + isTemporary, signature := getTemporaryAndSignature(params) + + isObs := getIsObs(isTemporary, querysResult, headers) + headers = copyHeaders(headers) + pathStyle := false + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + pathStyle = true + } + conf := &config{securityProviders: []securityProvider{NewBasicSecurityProvider(ak, sk, "")}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + + if isTemporary { + return getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature, conf, params, headers, isObs) + } + signature, region, signedHeaders := parseHeaders(headers) + if signature == "v4" { + conf.signature = SignatureV4 + requestURL, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestURL, _err := url.Parse(requestURL) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestURL") + return nil + } + headerKeys := strings.Split(signedHeaders, ";") + _headers := make(map[string][]string, len(headerKeys)) + for _, headerKey := range headerKeys { + _headers[headerKey] = headers[headerKey] + } + ret = v4Auth(ak, sk, region, method, canonicalizedURL, parsedRequestURL.RawQuery, _headers) + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else if signature == "v2" { + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedURL, headers, isObs) + v2HashPrefix := V2_HASH_PREFIX + if isObs { + v2HashPrefix = OBS_HASH_PREFIX + } + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + } + return + +} + +func getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature string, conf *config, params map[string]string, + headers map[string][]string, isObs bool) (ret map[string]string) { + + if signature == "v4" { + conf.signature = SignatureV4 + + longDate, ok := params[PARAM_DATE_AMZ_CAMEL] + if !ok { + longDate = params[HEADER_DATE_AMZ] + } + shortDate := longDate[:8] + + credential, ok := params[PARAM_CREDENTIAL_AMZ_CAMEL] + if !ok { + credential = params[strings.ToLower(PARAM_CREDENTIAL_AMZ_CAMEL)] + } + + _credential := UrlDecodeWithoutError(credential) + + regions := regionRegex.FindStringSubmatch(_credential) + var region string + if len(regions) >= 2 { + region = regions[1] + } + + _, scope := getCredential(ak, region, shortDate) + + expires, ok := params[PARAM_EXPIRES_AMZ_CAMEL] + if !ok { + expires = params[strings.ToLower(PARAM_EXPIRES_AMZ_CAMEL)] + } + + signedHeaders, ok := params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] + if !ok { + signedHeaders = params[strings.ToLower(PARAM_SIGNEDHEADERS_AMZ_CAMEL)] + } + + algorithm, ok := params[PARAM_ALGORITHM_AMZ_CAMEL] + if !ok { + algorithm = params[strings.ToLower(PARAM_ALGORITHM_AMZ_CAMEL)] + } + + if _, ok := params[PARAM_SIGNATURE_AMZ_CAMEL]; ok { + delete(params, PARAM_SIGNATURE_AMZ_CAMEL) + } else if _, ok := params[strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)]; ok { + delete(params, strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)) + } + + ret = make(map[string]string, 6) + ret[PARAM_ALGORITHM_AMZ_CAMEL] = algorithm + ret[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + ret[PARAM_DATE_AMZ_CAMEL] = longDate + ret[PARAM_EXPIRES_AMZ_CAMEL] = expires + ret[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = signedHeaders + + requestURL, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestURL, _err := url.Parse(requestURL) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl") + return nil + } + stringToSign := getV4StringToSign(method, canonicalizedURL, parsedRequestURL.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, strings.Split(signedHeaders, ";"), headers) + ret[PARAM_SIGNATURE_AMZ_CAMEL] = UrlEncode(getSignature(stringToSign, sk, region, shortDate), false) + } else if signature == "v2" { + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + expires, ok := params["Expires"] + if !ok { + expires = params["expires"] + } + headers[HEADER_DATE_CAMEL] = []string{expires} + stringToSign := getV2StringToSign(method, canonicalizedURL, headers, isObs) + ret = make(map[string]string, 3) + ret["Signature"] = UrlEncode(Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign))), false) + ret["AWSAccessKeyId"] = UrlEncode(ak, false) + ret["Expires"] = UrlEncode(expires, false) + } + + return +} diff --git a/vendor/golang.zx2c4.com/wireguard/wgctrl/wgtypes/types.go b/vendor/golang.zx2c4.com/wireguard/wgctrl/wgtypes/types.go index 32eba08d5f..7d17b62e99 100644 --- a/vendor/golang.zx2c4.com/wireguard/wgctrl/wgtypes/types.go +++ b/vendor/golang.zx2c4.com/wireguard/wgctrl/wgtypes/types.go @@ -18,7 +18,6 @@ const ( Unknown DeviceType = iota LinuxKernel OpenBSDKernel - WindowsKernel Userspace ) @@ -29,8 +28,6 @@ func (dt DeviceType) String() string { return "Linux kernel" case OpenBSDKernel: return "OpenBSD kernel" - case WindowsKernel: - return "Windows kernel" case Userspace: return "userspace" default: @@ -244,10 +241,6 @@ type PeerConfig struct { // from a device's peer list. Remove bool - // UpdateOnly specifies that an operation will only occur on this peer - // if the peer already exists as part of the interface. - UpdateOnly bool - // PresharedKey specifies a peer's preshared key configuration, if not nil. // // A non-nil, zero-value Key will clear the preshared key. diff --git a/vendor/modules.txt b/vendor/modules.txt index 5397689af7..d9604be327 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -485,6 +485,8 @@ github.com/golang/snappy # github.com/google/btree v1.0.0 ## explicit github.com/google/btree +# github.com/google/go-cmp v0.5.7 +## explicit; go 1.11 # github.com/google/go-querystring v1.0.0 ## explicit github.com/google/go-querystring/query @@ -543,6 +545,9 @@ github.com/huandu/xstrings # github.com/huaweicloud/huaweicloud-sdk-go v1.0.26 ## explicit github.com/huaweicloud/huaweicloud-sdk-go/auth/aksk +# github.com/huaweicloud/huaweicloud-sdk-go-obs v3.21.12+incompatible +## explicit +github.com/huaweicloud/huaweicloud-sdk-go-obs/obs # github.com/imdario/mergo v0.3.6 ## explicit github.com/imdario/mergo @@ -1107,8 +1112,8 @@ golang.org/x/tools/internal/imports ## explicit; go 1.11 golang.org/x/xerrors golang.org/x/xerrors/internal -# golang.zx2c4.com/wireguard/wgctrl v0.0.0-20220916014741-473347a5e6e3 -## explicit; go 1.17 +# golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987 +## explicit; go 1.13 golang.zx2c4.com/wireguard/wgctrl/wgtypes # google.golang.org/api v0.30.0 ## explicit; go 1.11