From 919f164f95d048912e733368dcba67c28e294278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=94=90=E6=96=8C?= Date: Thu, 3 Jan 2019 16:52:07 +0800 Subject: [PATCH 1/5] huawei support stage 2 --- cmd/huaweicli/main.go | 131 +++ pkg/cloudprovider/fakeregion.go | 2 +- pkg/cloudprovider/resources.go | 2 +- pkg/cloudprovider/waitstatus.go | 12 + pkg/compute/guestdrivers/huawei.go | 204 +++- pkg/compute/hostdrivers/managedvirtual.go | 8 +- pkg/compute/models/elasticips.go | 1 + pkg/compute/models/networks.go | 14 +- pkg/compute/models/storagecaches.go | 1 + pkg/compute/models/storages.go | 1 + pkg/compute/models/wires.go | 22 +- pkg/compute/models/zones.go | 154 ++- pkg/compute/tasks/eip_allocate_task.go | 2 +- pkg/mcclient/options/servers.go | 2 +- pkg/util/aliyun/eip.go | 2 +- pkg/util/aws/eip.go | 2 +- pkg/util/azure/eip.go | 2 +- pkg/util/httputils/httputils.go | 6 +- pkg/util/huawei/client/auth/signer.go | 4 +- pkg/util/huawei/client/client.go | 74 +- pkg/util/huawei/client/manager/manager.go | 36 +- .../huawei/client/modules/manager_base.go | 15 +- .../huawei/client/modules/manager_resource.go | 119 ++- .../huawei/client/modules/mod_balances.go | 59 ++ pkg/util/huawei/client/modules/mod_disks.go | 2 +- pkg/util/huawei/client/modules/mod_domains.go | 25 + pkg/util/huawei/client/modules/mod_flavors.go | 25 + pkg/util/huawei/client/modules/mod_images.go | 61 +- pkg/util/huawei/client/modules/mod_jobs.go | 57 ++ pkg/util/huawei/client/modules/mod_orders.go | 70 ++ pkg/util/huawei/client/modules/mod_port.go | 20 +- .../huawei/client/modules/mod_secgroups.go | 14 + pkg/util/huawei/client/modules/mod_servers.go | 51 +- .../huawei/client/modules/mod_snapshots.go | 19 +- pkg/util/huawei/client/modules/mod_users.go | 25 + pkg/util/huawei/consts.go | 18 + pkg/util/huawei/disk.go | 100 +- pkg/util/huawei/domain.go | 38 + pkg/util/huawei/eip.go | 144 ++- pkg/util/huawei/host.go | 143 ++- pkg/util/huawei/huawei.go | 51 +- pkg/util/huawei/image.go | 178 +++- pkg/util/huawei/instance.go | 629 +++++++++++- pkg/util/huawei/instancenic.go | 30 +- pkg/util/huawei/instancetype.go | 44 + pkg/util/huawei/keypair.go | 93 ++ pkg/util/huawei/latitud_and_longitude.go | 14 +- pkg/util/huawei/network.go | 22 +- pkg/util/huawei/obs/auth.go | 371 +++++++ pkg/util/huawei/obs/client.go | 904 +++++++++++++++++ pkg/util/huawei/obs/conf.go | 375 +++++++ pkg/util/huawei/obs/const.go | 493 ++++++++++ pkg/util/huawei/obs/convert.go | 699 +++++++++++++ pkg/util/huawei/obs/doc.go | 1 + pkg/util/huawei/obs/error.go | 21 + pkg/util/huawei/obs/http.go | 429 ++++++++ pkg/util/huawei/obs/log.go | 277 ++++++ pkg/util/huawei/obs/model.go | 931 ++++++++++++++++++ pkg/util/huawei/obs/temporary.go | 666 +++++++++++++ pkg/util/huawei/obs/trait.go | 701 +++++++++++++ pkg/util/huawei/obs/util.go | 394 ++++++++ pkg/util/huawei/order.go | 74 ++ pkg/util/huawei/provider/provider.go | 3 +- pkg/util/huawei/region.go | 261 ++++- pkg/util/huawei/securitygroup.go | 101 +- pkg/util/huawei/shell/business.go | 19 + pkg/util/huawei/shell/disk.go | 33 + pkg/util/huawei/shell/doc.go | 1 + pkg/util/huawei/shell/eip.go | 56 ++ pkg/util/huawei/shell/image.go | 32 + pkg/util/huawei/shell/instance.go | 140 +++ pkg/util/huawei/shell/instancetype.go | 22 + pkg/util/huawei/shell/keypair.go | 33 + pkg/util/huawei/shell/network.go | 22 + pkg/util/huawei/shell/printutils.go | 11 + pkg/util/huawei/shell/region.go | 16 + pkg/util/huawei/shell/secgroup.go | 34 + pkg/util/huawei/shell/snapshot.go | 43 + pkg/util/huawei/shell/vpc.go | 21 + pkg/util/huawei/shell/zone.go | 20 + pkg/util/huawei/snapshot.go | 22 +- pkg/util/huawei/storage.go | 16 +- pkg/util/huawei/storagecache.go | 204 +++- pkg/util/huawei/task.go | 80 ++ pkg/util/huawei/utils.go | 74 +- pkg/util/huawei/vpc.go | 50 +- pkg/util/huawei/wire.go | 83 +- pkg/util/huawei/zone.go | 11 + pkg/util/imagetools/imagetools.go | 12 +- pkg/util/openstack/region.go | 2 +- pkg/util/qcloud/eip.go | 2 +- 91 files changed, 10177 insertions(+), 331 deletions(-) create mode 100644 cmd/huaweicli/main.go create mode 100644 pkg/util/huawei/client/modules/mod_balances.go create mode 100644 pkg/util/huawei/client/modules/mod_domains.go create mode 100644 pkg/util/huawei/client/modules/mod_flavors.go create mode 100644 pkg/util/huawei/client/modules/mod_jobs.go create mode 100644 pkg/util/huawei/client/modules/mod_orders.go create mode 100644 pkg/util/huawei/client/modules/mod_users.go create mode 100644 pkg/util/huawei/domain.go create mode 100644 pkg/util/huawei/instancetype.go create mode 100644 pkg/util/huawei/obs/auth.go create mode 100644 pkg/util/huawei/obs/client.go create mode 100644 pkg/util/huawei/obs/conf.go create mode 100644 pkg/util/huawei/obs/const.go create mode 100644 pkg/util/huawei/obs/convert.go create mode 100644 pkg/util/huawei/obs/doc.go create mode 100644 pkg/util/huawei/obs/error.go create mode 100644 pkg/util/huawei/obs/http.go create mode 100644 pkg/util/huawei/obs/log.go create mode 100644 pkg/util/huawei/obs/model.go create mode 100644 pkg/util/huawei/obs/temporary.go create mode 100644 pkg/util/huawei/obs/trait.go create mode 100644 pkg/util/huawei/obs/util.go create mode 100644 pkg/util/huawei/order.go create mode 100644 pkg/util/huawei/shell/business.go create mode 100644 pkg/util/huawei/shell/disk.go create mode 100644 pkg/util/huawei/shell/doc.go create mode 100644 pkg/util/huawei/shell/eip.go create mode 100644 pkg/util/huawei/shell/image.go create mode 100644 pkg/util/huawei/shell/instance.go create mode 100644 pkg/util/huawei/shell/instancetype.go create mode 100644 pkg/util/huawei/shell/keypair.go create mode 100644 pkg/util/huawei/shell/network.go create mode 100644 pkg/util/huawei/shell/printutils.go create mode 100644 pkg/util/huawei/shell/region.go create mode 100644 pkg/util/huawei/shell/secgroup.go create mode 100644 pkg/util/huawei/shell/snapshot.go create mode 100644 pkg/util/huawei/shell/vpc.go create mode 100644 pkg/util/huawei/shell/zone.go create mode 100644 pkg/util/huawei/task.go diff --git a/cmd/huaweicli/main.go b/cmd/huaweicli/main.go new file mode 100644 index 0000000000..b92441ea47 --- /dev/null +++ b/cmd/huaweicli/main.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "os" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" + "yunion.io/x/structarg" + + _ "yunion.io/x/onecloud/pkg/util/huawei/shell" +) + +type BaseOptions struct { + Help bool `help:"Show help"` + AccessUrl string `help:"Access key" default:"$HUAWEI_ACCESS_URL" choices:"ChinaCloud|InternationalCloud"` + AccessKey string `help:"Access key" default:"$HUAWEI_ACCESS_KEY"` + Secret string `help:"Secret" default:"$HUAWEI_SECRET"` + RegionId string `help:"RegionId" default:"$HUAWEI_REGION"` + ProjectId string `help:"RegionId" default:"$HUAWEI_PROJECT"` + SUBCOMMAND string `help:"huaweicli subcommand" subcommand:"true"` +} + +func getSubcommandParser() (*structarg.ArgumentParser, error) { + parse, e := structarg.NewArgumentParser(&BaseOptions{}, + "huaweicli", + "Command-line interface to huawei API.", + `See "huaweicli help COMMAND" for help on a specific command.`) + + if e != nil { + return nil, e + } + + subcmd := parse.GetSubcommand() + if subcmd == nil { + return nil, fmt.Errorf("No subcommand argument.") + } + type HelpOptions struct { + SUBCOMMAND string `help:"sub-command name"` + } + shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error { + helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND) + if e != nil { + return e + } else { + fmt.Print(helpstr) + return nil + } + }) + for _, v := range shellutils.CommandTable { + _, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback) + if e != nil { + return nil, e + } + } + return parse, nil +} + +func showErrorAndExit(e error) { + log.Errorf("%s", e) + os.Exit(1) +} + +func newClient(options *BaseOptions) (*huawei.SRegion, error) { + if len(options.AccessKey) == 0 { + return nil, fmt.Errorf("Missing accessKey") + } + + if len(options.Secret) == 0 { + return nil, fmt.Errorf("Missing secret") + } + + account := "" + if len(options.ProjectId) > 0 { + account = options.AccessKey + "/" + options.ProjectId + } else { + account = options.AccessKey + } + + cli, err := huawei.NewHuaweiClient("", "", options.AccessUrl, account, options.Secret) + if err != nil { + return nil, err + } + + region := cli.GetRegion(options.RegionId) + if region == nil { + return nil, fmt.Errorf("No such region %s", options.RegionId) + } + + return region, nil +} + +func main() { + parser, e := getSubcommandParser() + if e != nil { + showErrorAndExit(e) + } + e = parser.ParseArgs(os.Args[1:], false) + options := parser.Options().(*BaseOptions) + + if options.Help { + fmt.Print(parser.HelpString()) + } else { + subcmd := parser.GetSubcommand() + subparser := subcmd.GetSubParser() + if e != nil { + if subparser != nil { + fmt.Print(subparser.Usage()) + } else { + fmt.Print(parser.Usage()) + } + showErrorAndExit(e) + } else { + suboptions := subparser.Options() + if options.SUBCOMMAND == "help" { + e = subcmd.Invoke(suboptions) + } else { + var region *huawei.SRegion + region, e = newClient(options) + if e != nil { + showErrorAndExit(e) + } + e = subcmd.Invoke(region, suboptions) + } + if e != nil { + showErrorAndExit(e) + } + } + } +} diff --git a/pkg/cloudprovider/fakeregion.go b/pkg/cloudprovider/fakeregion.go index 5ff1352a12..4c849c8e30 100644 --- a/pkg/cloudprovider/fakeregion.go +++ b/pkg/cloudprovider/fakeregion.go @@ -68,7 +68,7 @@ func (region *SFakeOnPremiseRegion) CreateIVpc(name string, desc string, cidr st return nil, ErrNotSupported } -func (region *SFakeOnPremiseRegion) CreateEIP(name string, bwMbps int, chargeType string) (ICloudEIP, error) { +func (region *SFakeOnPremiseRegion) CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (ICloudEIP, error) { return nil, ErrNotSupported } diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 7889fb0880..6f63f5e6d5 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -48,7 +48,7 @@ type ICloudRegion interface { SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error) - CreateEIP(name string, bwMbps int, chargeType string) (ICloudEIP, error) + CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (ICloudEIP, error) GetISnapshots() ([]ICloudSnapshot, error) GetISnapshotById(snapshotId string) (ICloudSnapshot, error) diff --git a/pkg/cloudprovider/waitstatus.go b/pkg/cloudprovider/waitstatus.go index 10a8393cc3..cbe4d8841d 100644 --- a/pkg/cloudprovider/waitstatus.go +++ b/pkg/cloudprovider/waitstatus.go @@ -37,3 +37,15 @@ func WaitDeleted(res ICloudResource, interval time.Duration, timeout time.Durati } return ErrTimeout } + +func WaitCreated(interval time.Duration, timeout time.Duration, callback func() bool) error { + startTime := time.Now() + for time.Now().Sub(startTime) < timeout { + ok := callback() + if ok { + return nil + } + time.Sleep(interval) + } + return ErrTimeout +} diff --git a/pkg/compute/guestdrivers/huawei.go b/pkg/compute/guestdrivers/huawei.go index 3ae730af55..ddf3443f59 100644 --- a/pkg/compute/guestdrivers/huawei.go +++ b/pkg/compute/guestdrivers/huawei.go @@ -3,12 +3,16 @@ package guestdrivers import ( "context" "fmt" + "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/seclib2" "yunion.io/x/pkg/utils" ) @@ -83,11 +87,207 @@ func (self *SHuaweiGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m } func (self *SHuaweiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - // todo: implement me + config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) + log.Debugf("RequestDeployGuestOnHost: %s", config) + + action, err := config.GetString("action") + if err != nil { + return err + } + + publicKey, _ := config.GetString("public_key") + + adminPublicKey, _ := config.GetString("admin_public_key") + projectPublicKey, _ := config.GetString("project_public_key") + oUserData, _ := config.GetString("user_data") + + userData := generateUserData(adminPublicKey, projectPublicKey, oUserData) + + resetPassword := jsonutils.QueryBoolean(config, "reset_password", false) + passwd, _ := config.GetString("password") + if resetPassword && len(passwd) == 0 { + passwd = seclib2.RandomPassword2(12) + } + + ihost, err := host.GetIHost() + if err != nil { + return err + } + + desc := SManagedVMCreateConfig{} + err = config.Unmarshal(&desc, "desc") + if err != nil { + return err + } + + if action == "create" { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + + nets := guest.GetNetworks() + net := nets[0].GetNetwork() + vpc := net.GetVpc() + iregion, err := host.GetIRegion() + if err != nil { + return nil, err + } + + secgroupCache := models.SecurityGroupCacheManager.Register(ctx, task.GetUserCred(), desc.SecGroupId, vpc.Id, vpc.CloudregionId, vpc.ManagerId) + if secgroupCache == nil { + return nil, fmt.Errorf("failed to registor secgroupCache for secgroup: %s, vpc: %s", desc.SecGroupId, vpc.Name) + } + + secgroupExtId, err := iregion.SyncSecurityGroup(secgroupCache.ExternalId, vpc.ExternalId, desc.SecGroupName, "", desc.SecRules) + if err != nil { + log.Errorf("SyncSecurityGroup fail %s", err) + return nil, err + } + if err := secgroupCache.SetExternalId(secgroupExtId); err != nil { + return nil, fmt.Errorf("failed to set externalId for secgroup %s externalId %s: error: %v", desc.SecGroupId, secgroupExtId, err) + } + + var createErr error + var iVM cloudprovider.ICloudVM + + var bc *billing.SBillingCycle + if desc.BillingCycle.IsValid() { + bc = &desc.BillingCycle + } + + if len(desc.InstanceType) > 0 { + iVM, createErr = ihost.CreateVM2(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.InstanceType, desc.ExternalNetworkId, + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgroupExtId, userData, bc) + } else { + iVM, createErr = ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, publicKey, secgroupExtId, userData, bc) + } + + if createErr != nil { + return nil, createErr + } + log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_RUNNING, time.Second*5, time.Second*1800) + if err != nil { + return nil, err + } + log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId()) + + iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) + if err != nil { + log.Errorf("cannot find vm %s", err) + return nil, err + } + + data := fetchIVMinfo(desc, iVM, guest.Id, "root", passwd, action) + + return data, nil + }) + } else if action == "deploy" { + iVM, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil || iVM == nil { + log.Errorf("cannot find vm %s", err) + return fmt.Errorf("cannot find vm") + } + + params := task.GetParams() + log.Debugf("Deploy VM params %s", params.String()) + + name, _ := params.GetString("name") + description, _ := params.GetString("description") + publicKey, _ := config.GetString("public_key") + deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + + if len(userData) > 0 { + err := iVM.UpdateUserData(userData) + if err != nil { + log.Errorf("update userdata fail %s", err) + } + } + + err := iVM.DeployVM(ctx, name, passwd, publicKey, deleteKeypair, description) + if err != nil { + return nil, err + } + + data := fetchIVMinfo(desc, iVM, guest.Id, "root", passwd, action) + + return data, nil + }) + } else if action == "rebuild" { + + iVM, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil || iVM == nil { + log.Errorf("cannot find vm %s", err) + return fmt.Errorf("cannot find vm") + } + + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + if len(userData) > 0 { + err := iVM.UpdateUserData(userData) + if err != nil { + log.Errorf("update userdata fail %s", err) + } + } + + diskId, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) + if err != nil { + return nil, err + } + + log.Debugf("VMrebuildRoot %s new diskID %s, wait status ready ...", iVM.GetGlobalId(), diskId) + + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return nil, err + } + log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId()) + + maxWaitSecs := 300 + waited := 0 + + for { + // hack, wait disk number consistent + idisks, err := iVM.GetIDisks() + if err != nil { + log.Errorf("fail to find VM idisks %s", err) + return nil, err + } + if len(idisks) < len(desc.DataDisks)+1 || idisks[0].GetGlobalId() != diskId { + if waited > maxWaitSecs { + log.Errorf("inconsistent disk number, wait timeout, must be something wrong on remote") + return nil, cloudprovider.ErrTimeout + } + if len(idisks) < len(desc.DataDisks)+1 { + log.Debugf("inconsistent disk number???? %d != %d", len(idisks), len(desc.DataDisks)+1) + } + if len(idisks) > 0 && idisks[0].GetGlobalId() != diskId { + log.Errorf("system disk id inconsistent %s != %s", idisks[0].GetGlobalId(), diskId) + } + time.Sleep(time.Second * 5) + waited += 5 + } else { + break + } + } + + data := fetchIVMinfo(desc, iVM, guest.Id, "root", passwd, action) + + return data, nil + }) + + } else { + log.Errorf("RequestDeployGuestOnHost: Action %s not supported", action) + return fmt.Errorf("Action %s not supported", action) + } + return nil } func (self *SHuaweiGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool { - // todo: implement me + months := bc.GetMonths() + if (months >= 1 && months <= 9) || (months == 12) || (months == 24) || (months == 36) { + return true + } + return false } diff --git a/pkg/compute/hostdrivers/managedvirtual.go b/pkg/compute/hostdrivers/managedvirtual.go index 43ac28e46d..2da17e3cf5 100644 --- a/pkg/compute/hostdrivers/managedvirtual.go +++ b/pkg/compute/hostdrivers/managedvirtual.go @@ -30,7 +30,13 @@ func (self *SManagedVirtualizationHostDriver) CheckAndSetCacheImage(ctx context. osArch, _ := params.GetString("os_arch") osType, _ := params.GetString("os_type") osDist, _ := params.GetString("os_distribution") - osVersion, _ := params.GetString("os_version") + var osVersion string + providerName := storageCache.GetProviderName() + if providerName == models.CLOUD_PROVIDER_HUAWEI { + osVersion, _ = params.GetString("os_full_version") + } else { + osVersion, _ = params.GetString("os_version") + } isForce := jsonutils.QueryBoolean(params, "is_force", false) userCred := task.GetUserCred() diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index bb72366cae..84a6851905 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -77,6 +77,7 @@ type SElasticip struct { Bandwidth int `list:"user" create:"required"` ChargeType string `list:"user" create:"required"` + BgpType string `list:"user" create:"optional"` // 目前只有华为云此字段是必需填写的。 AutoDellocate tristate.TriState `default:"false" get:"user" create:"optional"` diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index de478e5cb1..6dd17c64bd 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -1045,7 +1045,19 @@ func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred } vpc := vpcObj.(*SVpc) zone := zoneObj.(*SZone) - wires, err := WireManager.getWiresByVpcAndZone(vpc, zone) + region := zone.GetRegion() + if region == nil { + return nil, httperrors.NewInternalServerError("zone %s related region not found", zone.Id) + } + + // 华为云wire zone_id 为空 + var wires []SWire + if region.Provider == CLOUD_PROVIDER_HUAWEI { + wires, err = WireManager.getWiresByVpcAndZone(vpc, nil) + } else { + wires, err = WireManager.getWiresByVpcAndZone(vpc, zone) + } + if err != nil { if err == sql.ErrNoRows { return nil, httperrors.NewNotFoundError("wire not found for zone %s and vpc %s", zoneStr, vpcStr) diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index 534ff8234a..9f4c8a2240 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -255,6 +255,7 @@ func (self *SStoragecache) StartImageCacheTask(ctx context.Context, userCred mcc data.Add(jsonutils.NewString(imgInfo.OsArch), "os_arch") data.Add(jsonutils.NewString(imgInfo.OsDistro), "os_distribution") data.Add(jsonutils.NewString(imgInfo.OsVersion), "os_version") + data.Add(jsonutils.NewString(imgInfo.OsFullVersion), "os_full_version") } if isForce { diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 47c5f0a44e..e8a028f2bc 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -90,6 +90,7 @@ var ( STORAGE_STANDARD_LRS, STORAGE_STANDARDSSD_LRS, STORAGE_PREMIUM_LRS, STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD, STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PREMIUM, + STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA, } STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 8c3a057d27..cc4014a5fe 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -40,9 +40,9 @@ func init() { type SWire struct { db.SStandaloneResourceBase - Bandwidth int `list:"admin" update:"admin" nullable:"false" create:"admin_required"` // = Column(Integer, nullable=False) # bandwidth of network in Mbps - ScheduleRank int `list:"admin" update:"admin"` // = Column(Integer, default=0, nullable=True) - ZoneId string `width:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` // = Column(VARCHAR(36, charset='ascii'), nullable=False) + Bandwidth int `list:"admin" update:"admin" nullable:"false" create:"admin_required"` // = Column(Integer, nullable=False) # bandwidth of network in Mbps + ScheduleRank int `list:"admin" update:"admin"` // = Column(Integer, default=0, nullable=True) + ZoneId string `width:"36" charset:"ascii" nullable:"true" list:"admin" create:"admin_required"` // = Column(VARCHAR(36, charset='ascii'), nullable=False) VpcId string `wdith:"36" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` } @@ -259,16 +259,20 @@ func (manager *SWireManager) newFromCloudWire(extWire cloudprovider.ICloudWire, wire.ExternalId = extWire.GetGlobalId() wire.Bandwidth = extWire.GetBandwidth() wire.VpcId = vpc.Id - zoneObj, err := ZoneManager.FetchByExternalId(extWire.GetIZone().GetGlobalId()) - if err != nil { - log.Errorf("cannot find zone for wire %s", err) - return nil, err + izone := extWire.GetIZone() + if izone != nil { + zoneObj, err := ZoneManager.FetchByExternalId(izone.GetGlobalId()) + if err != nil { + log.Errorf("cannot find zone for wire %s", err) + return nil, err + } + + wire.ZoneId = zoneObj.(*SZone).Id } - wire.ZoneId = zoneObj.(*SZone).Id wire.IsEmulated = extWire.IsEmulated() - err = manager.TableSpec().Insert(&wire) + err := manager.TableSpec().Insert(&wire) if err != nil { log.Errorf("newFromCloudWire fail %s", err) return nil, err diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index 3e65ec5d3e..bdcc47b688 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -479,6 +479,116 @@ func (manager *SZoneManager) InitializeData() error { return nil } +/* +Query 1: +vpc.manager_id is not empty && wire.zone_id is not empty +*/ +func (manager *SZoneManager) usableZoneQ1(providers, vpcs, wires, networks *sqlchemy.SSubQuery, usableNet, usableVpc bool) *sqlchemy.SSubQuery { + // join tables + sq := wires.Query(sqlchemy.DISTINCT("zone_id", wires.Field("zone_id"))) + if usableNet { + sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) + } + sq = sq.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))) + sq = sq.Join(providers, sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id"))) + + // add filters + if usableNet { + sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), NETWORK_STATUS_AVAILABLE)) + } + sq = sq.Filter(sqlchemy.IsNotEmpty(wires.Field("zone_id"))) + sq = sq.Filter(sqlchemy.IsTrue(providers.Field("enabled"))) + sq = sq.Filter(sqlchemy.In(providers.Field("status"), CLOUD_PROVIDER_VALID_STATUS)) + if usableVpc { + sq = sq.Filter(sqlchemy.Equals(vpcs.Field("status"), VPC_STATUS_AVAILABLE)) + } + + return sq.SubQuery() +} + +/* +Query 2: +vpc.manager_id is empty && wire.zone_id is not empty +*/ +func (manager *SZoneManager) usableZoneQ2(vpcs, wires, networks *sqlchemy.SSubQuery, usableNet, usableVpc bool) *sqlchemy.SSubQuery { + // join tables + sq := wires.Query(sqlchemy.DISTINCT("zone_id", wires.Field("zone_id"))) + if usableNet { + sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) + } + sq = sq.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))) + + // add filters + if usableNet { + sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), NETWORK_STATUS_AVAILABLE)) + } + sq = sq.Filter(sqlchemy.IsNotEmpty(wires.Field("zone_id"))) + sq = sq.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id"))) + if usableVpc { + sq = sq.Filter(sqlchemy.Equals(vpcs.Field("status"), VPC_STATUS_AVAILABLE)) + } + + return sq.SubQuery() +} + +/* +Query 3: +vpc.manager_id is not empty && wire.zone_id is empty + +2019.01.17 目前华为云子网在整个region 可用。wire中zone_id留空。 +*/ +func (manager *SZoneManager) usableZoneQ3(providers, vpcs, wires, networks, zones *sqlchemy.SSubQuery, usableNet, usableVpc bool) *sqlchemy.SSubQuery { + // join tables + sq := zones.Query(sqlchemy.DISTINCT("zone_id", zones.Field("id"))) + sq = sq.Join(vpcs, sqlchemy.Equals(zones.Field("cloudregion_id"), vpcs.Field("cloudregion_id"))) + sq = sq.Join(wires, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))) + if usableNet { + sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) + } + sq = sq.Join(providers, sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id"))) + + // add filters + if usableNet { + sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), NETWORK_STATUS_AVAILABLE)) + } + sq = sq.Filter(sqlchemy.IsNullOrEmpty(wires.Field("zone_id"))) + sq = sq.Filter(sqlchemy.IsTrue(providers.Field("enabled"))) + sq = sq.Filter(sqlchemy.In(providers.Field("status"), CLOUD_PROVIDER_VALID_STATUS)) + if usableVpc { + sq = sq.Filter(sqlchemy.Equals(vpcs.Field("status"), VPC_STATUS_AVAILABLE)) + } + + return sq.SubQuery() +} + +/* +Query 4: +vpc.manager_id is empty && wire.zone_id is empty + +2019.01.17 目前华为云子网在整个region 可用。wire中zone_id留空。 +*/ +func (manager *SZoneManager) usableZoneQ4(vpcs, wires, networks, zones *sqlchemy.SSubQuery, usableNet, usableVpc bool) *sqlchemy.SSubQuery { + // join tables + sq := zones.Query(sqlchemy.DISTINCT("zone_id", zones.Field("id"))) + sq = sq.Join(vpcs, sqlchemy.Equals(zones.Field("cloudregion_id"), vpcs.Field("cloudregion_id"))) + sq = sq.Join(wires, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))) + if usableNet { + sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) + } + + // add filters + if usableNet { + sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), NETWORK_STATUS_AVAILABLE)) + } + sq = sq.Filter(sqlchemy.IsNullOrEmpty(wires.Field("zone_id"))) + sq = sq.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id"))) + if usableVpc { + sq = sq.Filter(sqlchemy.Equals(vpcs.Field("status"), VPC_STATUS_AVAILABLE)) + } + + return sq.SubQuery() +} + func (manager *SZoneManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { q, err := manager.SStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { @@ -495,45 +605,25 @@ func (manager *SZoneManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu } if jsonutils.QueryBoolean(query, "usable", false) || jsonutils.QueryBoolean(query, "usable_vpc", false) { + usableNet := jsonutils.QueryBoolean(query, "usable", false) + usableVpc := jsonutils.QueryBoolean(query, "usable_vpc", false) + networks := NetworkManager.Query().SubQuery() wires := WireManager.Query().SubQuery() vpcs := VpcManager.Query().SubQuery() providers := CloudproviderManager.Query().SubQuery() + zones := ZoneManager.Query().SubQuery() - usableNet := jsonutils.QueryBoolean(query, "usable", false) - usableVpc := jsonutils.QueryBoolean(query, "usable_vpc", false) - - sq := wires.Query(sqlchemy.DISTINCT("zone_id", wires.Field("zone_id"))) - if usableNet { - sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) - } - sq = sq.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))) - sq = sq.Join(providers, sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id"))) - if usableNet { - sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), NETWORK_STATUS_AVAILABLE)) - } - sq = sq.Filter(sqlchemy.IsTrue(providers.Field("enabled"))) - sq = sq.Filter(sqlchemy.In(providers.Field("status"), CLOUD_PROVIDER_VALID_STATUS)) - if usableVpc { - sq = sq.Filter(sqlchemy.Equals(vpcs.Field("status"), VPC_STATUS_AVAILABLE)) - } - - sq2 := wires.Query(sqlchemy.DISTINCT("zone_id", wires.Field("zone_id"))) - if usableNet { - sq2 = sq2.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) - } - sq2 = sq2.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id"))) - if usableNet { - sq2 = sq2.Filter(sqlchemy.Equals(networks.Field("status"), NETWORK_STATUS_AVAILABLE)) - } - sq2 = sq2.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id"))) - if usableVpc { - sq2 = sq2.Filter(sqlchemy.Equals(vpcs.Field("status"), VPC_STATUS_AVAILABLE)) - } + sq1 := manager.usableZoneQ1(providers, vpcs, wires, networks, usableNet, usableVpc) + sq2 := manager.usableZoneQ2(vpcs, wires, networks, usableNet, usableVpc) + sq3 := manager.usableZoneQ3(providers, vpcs, wires, networks, zones, usableNet, usableVpc) + sq4 := manager.usableZoneQ4(vpcs, wires, networks, zones, usableNet, usableVpc) q = q.Filter(sqlchemy.OR( - sqlchemy.In(q.Field("id"), sq.SubQuery()), - sqlchemy.In(q.Field("id"), sq2.SubQuery()), + sqlchemy.In(q.Field("id"), sq1), + sqlchemy.In(q.Field("id"), sq2), + sqlchemy.In(q.Field("id"), sq3), + sqlchemy.In(q.Field("id"), sq4), )) q = q.Equals("status", ZONE_ENABLE) } diff --git a/pkg/compute/tasks/eip_allocate_task.go b/pkg/compute/tasks/eip_allocate_task.go index 1f90eaa240..22bcf0a43d 100644 --- a/pkg/compute/tasks/eip_allocate_task.go +++ b/pkg/compute/tasks/eip_allocate_task.go @@ -45,7 +45,7 @@ func (self *EipAllocateTask) OnInit(ctx context.Context, obj db.IStandaloneModel return } - extEip, err := iregion.CreateEIP(eip.Name, eip.Bandwidth, eip.ChargeType) + extEip, err := iregion.CreateEIP(eip.Name, eip.Bandwidth, eip.ChargeType, "") if err != nil { msg := fmt.Sprintf("create eip fail %s", err) eip.SetStatus(self.UserCred, models.EIP_STATUS_ALLOCATE_FAIL, msg) diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index ec4d589d6b..610c559f7d 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -18,7 +18,7 @@ type ServerListOptions struct { Gpu *bool `help:"Show gpu servers"` Secgroup string `help:"Secgroup ID or Name"` AdminSecgroup string `help:"AdminSecgroup ID or Name"` - Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws"` + Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws|huawei"` Region string `help:"Show servers in cloudregion"` WithEip *bool `help:"Show Servers with EIP"` WithoutEip *bool `help:"Show Servers without EIP"` diff --git a/pkg/util/aliyun/eip.go b/pkg/util/aliyun/eip.go index 7c77a4f1b3..3f32e82b4f 100644 --- a/pkg/util/aliyun/eip.go +++ b/pkg/util/aliyun/eip.go @@ -242,7 +242,7 @@ func (region *SRegion) AllocateEIP(bwMbps int, chargeType TInternetChargeType) ( return region.GetEip(eipId) } -func (region *SRegion) CreateEIP(name string, bwMbps int, chargeType string) (cloudprovider.ICloudEIP, error) { +func (region *SRegion) CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (cloudprovider.ICloudEIP, error) { var ctype TInternetChargeType switch chargeType { case models.EIP_CHARGE_TYPE_BY_TRAFFIC: diff --git a/pkg/util/aws/eip.go b/pkg/util/aws/eip.go index f732258950..1e76df40fd 100644 --- a/pkg/util/aws/eip.go +++ b/pkg/util/aws/eip.go @@ -236,7 +236,7 @@ func (self *SRegion) AllocateEIP(domainType string) (*SEipAddress, error) { return self.GetEip(*eip.AllocationId) } -func (self *SRegion) CreateEIP(name string, bwMbps int, chargeType string) (cloudprovider.ICloudEIP, error) { +func (self *SRegion) CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (cloudprovider.ICloudEIP, error) { // todo: aws 不支持指定bwMbps, chargeType ？ log.Debugf("CreateEip: aws not support specific params name/bwMbps/chargeType.") ieip, err := self.AllocateEIP("vpc") diff --git a/pkg/util/azure/eip.go b/pkg/util/azure/eip.go index 88fc697bc2..4b5db78ad4 100644 --- a/pkg/util/azure/eip.go +++ b/pkg/util/azure/eip.go @@ -68,7 +68,7 @@ func (region *SRegion) AllocateEIP(eipName string) (*SEipAddress, error) { return &eip, cloudprovider.WaitStatus(&eip, models.EIP_STATUS_READY, 10*time.Second, 300*time.Second) } -func (region *SRegion) CreateEIP(eipName string, bwMbps int, chargeType string) (cloudprovider.ICloudEIP, error) { +func (region *SRegion) CreateEIP(eipName string, bwMbps int, chargeType string, bgpType string) (cloudprovider.ICloudEIP, error) { return region.AllocateEIP(eipName) } diff --git a/pkg/util/httputils/httputils.go b/pkg/util/httputils/httputils.go index fb0fb3c988..2194abddb5 100644 --- a/pkg/util/httputils/httputils.go +++ b/pkg/util/httputils/httputils.go @@ -248,8 +248,10 @@ func ParseJSONResponse(resp *http.Response, err error, debug bool) (http.Header, } err = jrbody.Unmarshal(&ce) - if err != nil { - return nil, nil, err + if err != nil || ce.Code == 0 { + ce.Code = resp.StatusCode + ce.Details = jrbody.String() + return nil, nil, &ce } else { return nil, nil, &ce } diff --git a/pkg/util/huawei/client/auth/signer.go b/pkg/util/huawei/client/auth/signer.go index ead1498fce..73f37d0f88 100644 --- a/pkg/util/huawei/client/auth/signer.go +++ b/pkg/util/huawei/client/auth/signer.go @@ -72,7 +72,9 @@ func signRequest(request requests.IRequest, signer Signer) error { func fillRequiredHeaders(request requests.IRequest, t time.Time) { request.AddHeaderParam("HOST", request.GetHost()) request.AddHeaderParam("X-Sdk-Date", formattedSignTime(t, "DateTime")) - request.AddHeaderParam("X-Project-Id", request.GetProjectId()) + if len(request.GetProjectId()) > 0 { + request.AddHeaderParam("X-Project-Id", request.GetProjectId()) + } return } diff --git a/pkg/util/huawei/client/client.go b/pkg/util/huawei/client/client.go index 3ccf6cd659..19361ca77a 100644 --- a/pkg/util/huawei/client/client.go +++ b/pkg/util/huawei/client/client.go @@ -12,20 +12,33 @@ type Client struct { domainId string projectId string + // 标记初始化状态 + init bool + + Balances *modules.SBalanceManager Bandwidths *modules.SBandwidthManager Disks *modules.SDiskManager + Domains *modules.SDomainManager Eips *modules.SEipManager + Flavors *modules.SFlavorManager Images *modules.SImageManager + OpenStackImages *modules.SImageManager Interface *modules.SInterfaceManager + Jobs *modules.SJobManager Keypairs *modules.SKeypairManager + Orders *modules.SOrderManager Port *modules.SPortManager Projects *modules.SProjectManager Regions *modules.SRegionManager SecurityGroupRules *modules.SSecgroupRuleManager SecurityGroups *modules.SSecurityGroupManager + NovaSecurityGroups *modules.SSecurityGroupManager Servers *modules.SServerManager + NovaServers *modules.SServerManager Snapshots *modules.SSnapshotManager + OsSnapshots *modules.SSnapshotManager Subnets *modules.SSubnetManager + Users *modules.SUserManager Vpcs *modules.SVpcManager Zones *modules.SZoneManager } @@ -62,69 +75,36 @@ func (self *Client) InitWithAccessKey(regionId, projectId, accessKey, secretKey } func (self *Client) initManagers() { - if self.Servers == nil { + if !self.init { self.Servers = modules.NewServerManager(self.regionId, self.projectId, self.signer) - } - - if self.Snapshots == nil { + self.NovaServers = modules.NewNovaServerManager(self.regionId, self.projectId, self.signer) self.Snapshots = modules.NewSnapshotManager(self.regionId, self.projectId, self.signer) - } - - if self.Images == nil { - self.Images = modules.NewImageManager(self.regionId, self.signer) - } - - if self.Projects == nil { + self.OsSnapshots = modules.NewOsSnapshotManager(self.regionId, self.projectId, self.signer) + self.Images = modules.NewImageManager(self.regionId, self.projectId, self.signer) + self.OpenStackImages = modules.NewOpenstackImageManager(self.regionId, self.signer) self.Projects = modules.NewProjectManager(self.signer) - } - - if self.Regions == nil { self.Regions = modules.NewRegionManager(self.signer) - } - - if self.Zones == nil { self.Zones = modules.NewZoneManager(self.regionId, self.projectId, self.signer) - } - - if self.Vpcs == nil { self.Vpcs = modules.NewVpcManager(self.regionId, self.projectId, self.signer) - } - - if self.Eips == nil { self.Eips = modules.NewEipManager(self.regionId, self.projectId, self.signer) - } - - if self.Disks == nil { self.Disks = modules.NewDiskManager(self.regionId, self.projectId, self.signer) - } - - if self.Keypairs == nil { + self.Domains = modules.NewDomainManager(self.signer) self.Keypairs = modules.NewKeypairManager(self.regionId, self.projectId, self.signer) - } - - if self.SecurityGroupRules == nil { + self.Orders = modules.NewOrderManager(self.regionId, self.signer) self.SecurityGroupRules = modules.NewSecgroupRuleManager(self.regionId, self.projectId, self.signer) - } - - if self.SecurityGroups == nil { self.SecurityGroups = modules.NewSecurityGroupManager(self.regionId, self.projectId, self.signer) - } - - if self.Subnets == nil { + self.NovaSecurityGroups = modules.NewNovaSecurityGroupManager(self.regionId, self.projectId, self.signer) self.Subnets = modules.NewSubnetManager(self.regionId, self.projectId, self.signer) - } - - if self.Interface == nil { + self.Users = modules.NewUserManager(self.signer) self.Interface = modules.NewInterfaceManager(self.regionId, self.projectId, self.signer) - } - - if self.Bandwidths == nil { + self.Jobs = modules.NewJobManager(self.regionId, self.projectId, self.signer) + self.Balances = modules.NewBalanceManager(self.signer) self.Bandwidths = modules.NewBandwidthManager(self.regionId, self.projectId, self.signer) + self.Port = modules.NewPortManager(self.regionId, self.projectId, self.signer) + self.Flavors = modules.NewFlavorManager(self.regionId, self.projectId, self.signer) } - if self.Port == nil { - self.Port = modules.NewPortManager(self.regionId, self.signer) - } + self.init = true } // todo: init from envrioment diff --git a/pkg/util/huawei/client/manager/manager.go b/pkg/util/huawei/client/manager/manager.go index c44bea432e..f82e76de81 100644 --- a/pkg/util/huawei/client/manager/manager.go +++ b/pkg/util/huawei/client/manager/manager.go @@ -1,25 +1,12 @@ package manager import ( - "fmt" - "net/url" - "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/util/huawei/client/responses" ) -type ManagerContext struct { - InstanceManager IManager - InstanceId string -} - -func (self *ManagerContext) GetPath() string { - path := self.InstanceManager.KeyString() - if len(self.InstanceId) > 0 { - path += fmt.Sprintf("/%s", url.PathEscape(self.InstanceId)) - } - - return path +type IManagerContext interface { + GetPath() string } type IBaseManager interface { @@ -34,30 +21,35 @@ type IManager interface { // 获取资源列表 GET /cloudservers/? List(querys map[string]string) (*responses.ListResult, error) // 根据上文获取资源列表 GET /cloudservers//nics? - ListInContext(ctx *ManagerContext, spec string, querys map[string]string) (*responses.ListResult, error) + ListInContext(ctx IManagerContext, querys map[string]string) (*responses.ListResult, error) + ListInContextWithSpec(ctx IManagerContext, spec string, querys map[string]string, responseKey string) (*responses.ListResult, error) // 查询单个资源 GET /cloudservers/? Get(id string, querys map[string]string) (jsonutils.JSONObject, error) // 根据上文获取资源查询单个资源 GET /cloudservers//nics/? - GetInContext(ctx *ManagerContext, id string, querys map[string]string) (jsonutils.JSONObject, error) + GetInContext(ctx IManagerContext, id string, querys map[string]string) (jsonutils.JSONObject, error) // 创建单个资源 POST /cloudservers - Create(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) + Create(params jsonutils.JSONObject) (jsonutils.JSONObject, error) // 根据上文创建单个资源 POST /cloudservers//nics/ - CreateInContext(ctx *ManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) + CreateInContext(ctx IManagerContext, params jsonutils.JSONObject) (jsonutils.JSONObject, error) + // 异步任务创建 POST /cloudservers. 返回异步任务 job_id。 todo：// 后续考虑返回一个task对象 + AsyncCreate(params jsonutils.JSONObject) (string, error) // 更新单个资源 PUT /cloudservers/ Update(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) // 根据上文更新单个资源 PUT /cloudservers//nics/ - UpdateInContext(ctx *ManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) + UpdateInContext(ctx IManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) + // 根据上文和spec更新单个资源 PUT /v2.1/{project_id}/servers/{server_id}/os-reset-password + UpdateInContextWithSpec(ctx IManagerContext, id string, spec string, params jsonutils.JSONObject, responseKey string) (jsonutils.JSONObject, error) // 删除单个资源 DELETE /cloudservers/ Delete(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) // 根据上文删除单个资源 DELETE /cloudservers//nics/ - DeleteInContext(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) + DeleteInContext(ctx IManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) // 批量执行操作 POST /cloudservers/ - BatchPerformAction(action string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) + // BatchPerformAction(action string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) // 执行操作 POST /cloudservers// PerformAction(action string, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) } diff --git a/pkg/util/huawei/client/modules/manager_base.go b/pkg/util/huawei/client/modules/manager_base.go index 7d6091a2bb..93fe77d15a 100644 --- a/pkg/util/huawei/client/modules/manager_base.go +++ b/pkg/util/huawei/client/modules/manager_base.go @@ -13,9 +13,14 @@ import ( "yunion.io/x/onecloud/pkg/util/huawei/client/responses" ) +type IRequestHook interface { + Process(r requests.IRequest) +} + type BaseManager struct { - signer auth.Signer - httpClient *http.Client + signer auth.Signer + httpClient *http.Client + requestHook IRequestHook // 用于对request做特殊处理。非必要请不要使用！！！。目前只有port接口用到。 columns []string debug bool @@ -90,6 +95,10 @@ func (self *BaseManager) _get(request requests.IRequest, responseKey string) (js func (self *BaseManager) jsonRequest(request requests.IRequest) (http.Header, jsonutils.JSONObject, error) { ctx := context.Background() + // hook request + if self.requestHook != nil { + self.requestHook.Process(request) + } // 拼接、编译、签名 requests here。 err := self.buildRequestWithSigner(request, self.signer) if err != nil { @@ -109,7 +118,7 @@ func (self *BaseManager) jsonRequest(request requests.IRequest) (http.Header, js } } - // 发送 request。 + // 发送 request。todo: 支持debug return httputils.JSONRequest(self.httpClient, ctx, httputils.THttpMethod(request.GetMethod()), request.BuildUrl(), header, jsonBody, self.debug) } diff --git a/pkg/util/huawei/client/modules/manager_resource.go b/pkg/util/huawei/client/modules/manager_resource.go index 0c1c3c173b..448236580c 100644 --- a/pkg/util/huawei/client/modules/manager_resource.go +++ b/pkg/util/huawei/client/modules/manager_resource.go @@ -1,6 +1,7 @@ package modules import ( + "fmt" "net/url" "strings" @@ -28,9 +29,24 @@ const ( ServiceNameOBS ServiceNameType = "obs" // 对象存储服务 OBS ServiceNameVPC ServiceNameType = "vpc" // 虚拟私有云 VPC ServiceNameELB ServiceNameType = "elb" // 弹性负载均衡 ELB + ServiceNameBSS ServiceNameType = "bss" // 合作伙伴运营能力 ) +type ManagerContext struct { + InstanceManager manager.IManager + InstanceId string +} + +func (self *ManagerContext) GetPath() string { + path := self.InstanceManager.KeyString() + if len(self.InstanceId) > 0 { + path += fmt.Sprintf("/%s", url.PathEscape(self.InstanceId)) + } + + return path +} + type ResourceManager struct { BaseManager ServiceName ServiceNameType // 服务名称： ecs @@ -43,6 +59,14 @@ type ResourceManager struct { ResourceKeyword string // 资源名称。url中使用 } +func getContent(params jsonutils.JSONObject) string { + if params == nil { + return "" + } + + return params.String() +} + func (self *ResourceManager) Version() string { return self.version } @@ -59,7 +83,7 @@ func (self *ResourceManager) GetColumns() []string { return []string{} } -func (self *ResourceManager) getReourcePath(ctx *manager.ManagerContext, rid string, spec string) string { +func (self *ResourceManager) getReourcePath(ctx manager.IManagerContext, rid string, spec string) string { segs := []string{} if ctx != nil { segs = append(segs, ctx.GetPath()) @@ -72,81 +96,114 @@ func (self *ResourceManager) getReourcePath(ctx *manager.ManagerContext, rid str } if len(spec) > 0 { - segs = append(segs, url.PathEscape(spec)) + specSegs := strings.Split(spec, "/") + for _, specSeg := range specSegs { + segs = append(segs, url.PathEscape(specSeg)) + } } return strings.Join(segs, "/") } -func (self *ResourceManager) newRequest(method, rid, spec string, ctx *manager.ManagerContext) *requests.SRequest { +func (self *ResourceManager) newRequest(method, rid, spec string, ctx manager.IManagerContext) *requests.SRequest { resourcePath := self.getReourcePath(ctx, rid, spec) return requests.NewResourceRequest(method, string(self.ServiceName), self.version, self.Region, self.ProjectId, resourcePath) } func (self *ResourceManager) List(querys map[string]string) (*responses.ListResult, error) { - return self.ListInContext(nil, "", querys) + return self.ListInContext(nil, querys) } -func (self *ResourceManager) ListInContext(ctx *manager.ManagerContext, spec string, querys map[string]string) (*responses.ListResult, error) { +func (self *ResourceManager) ListInContext(ctx manager.IManagerContext, querys map[string]string) (*responses.ListResult, error) { + return self.ListInContextWithSpec(ctx, "", querys, self.KeywordPlural) +} + +func (self *ResourceManager) ListInContextWithSpec(ctx manager.IManagerContext, spec string, querys map[string]string, responseKey string) (*responses.ListResult, error) { request := self.newRequest("GET", "", spec, ctx) for k, v := range querys { request.AddQueryParam(k, v) } - //if _, ok := request.GetQueryParams()["limit"]; !ok { - // request.AddQueryParam("limit", "50") - //} - - //if _, ok := request.GetQueryParams()["offset"]; !ok { - // request.AddQueryParam("offset", "0") - //} - - return self._list(request, self.KeywordPlural) + return self._list(request, responseKey) } func (self *ResourceManager) Get(id string, querys map[string]string) (jsonutils.JSONObject, error) { return self.GetInContext(nil, id, querys) } -func (self *ResourceManager) GetInContext(ctx *manager.ManagerContext, id string, querys map[string]string) (jsonutils.JSONObject, error) { - request := self.newRequest("GET", id, "", nil) +func (self *ResourceManager) GetInContext(ctx manager.IManagerContext, id string, querys map[string]string) (jsonutils.JSONObject, error) { + return self.GetInContextWithSpec(ctx, id, "", querys, self.Keyword) +} + +func (self *ResourceManager) GetInContextWithSpec(ctx manager.IManagerContext, id string, spec string, querys map[string]string, responseKey string) (jsonutils.JSONObject, error) { + request := self.newRequest("GET", id, spec, ctx) for k, v := range querys { request.AddQueryParam(k, v) } - return self._get(request, self.Keyword) + return self._get(request, responseKey) } -func (self *ResourceManager) Create(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") +func (self *ResourceManager) Create(params jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return self.CreateInContext(nil, params) } -func (self *ResourceManager) CreateInContext(ctx *manager.ManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") +func (self *ResourceManager) CreateInContext(ctx manager.IManagerContext, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return self.CreateInContextWithSpec(ctx, "", params, self.Keyword) +} + +func (self *ResourceManager) CreateInContextWithSpec(ctx manager.IManagerContext, spec string, params jsonutils.JSONObject, responseKey string) (jsonutils.JSONObject, error) { + request := self.newRequest("POST", "", spec, ctx) + request.SetContent([]byte(params.String())) + + return self._do(request, responseKey) +} + +func (self *ResourceManager) AsyncCreate(params jsonutils.JSONObject) (string, error) { + return "", fmt.Errorf("not supported") } func (self *ResourceManager) Update(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") + return self.UpdateInContext(nil, id, params) } -func (self *ResourceManager) UpdateInContext(ctx *manager.ManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") +func (self *ResourceManager) UpdateInContext(ctx manager.IManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return self.UpdateInContextWithSpec(ctx, id, "", params, self.Keyword) +} + +func (self *ResourceManager) UpdateInContextWithSpec(ctx manager.IManagerContext, id string, spec string, params jsonutils.JSONObject, responseKey string) (jsonutils.JSONObject, error) { + request := self.newRequest("PUT", id, spec, ctx) + content := getContent(params) + if len(content) > 0 { + request.SetContent([]byte(content)) + } + + return self._do(request, responseKey) } func (self *ResourceManager) Delete(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") + return self.DeleteInContext(nil, id, params) } -func (self *ResourceManager) DeleteInContext(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") -} +func (self *ResourceManager) DeleteInContext(ctx manager.IManagerContext, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { + request := self.newRequest("DELETE", id, "", ctx) + content := getContent(params) + if len(content) > 0 { + request.SetContent([]byte(content)) + } -func (self *ResourceManager) BatchPerformAction(action string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") + return self._do(request, self.Keyword) } func (self *ResourceManager) PerformAction(action string, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - panic("implement me") + return self.PerformAction2(action, id, params, self.Keyword) +} + +func (self *ResourceManager) PerformAction2(action string, id string, params jsonutils.JSONObject, responseKey string) (jsonutils.JSONObject, error) { + request := self.newRequest("POST", id, action, nil) + request.SetContent([]byte(getContent(params))) + + return self._do(request, responseKey) } func (self *ResourceManager) SetVersion(v string) { diff --git a/pkg/util/huawei/client/modules/mod_balances.go b/pkg/util/huawei/client/modules/mod_balances.go new file mode 100644 index 0000000000..84ebd07466 --- /dev/null +++ b/pkg/util/huawei/client/modules/mod_balances.go @@ -0,0 +1,59 @@ +package modules + +import ( + "fmt" + "net/http" + + "yunion.io/x/onecloud/pkg/util/huawei/client/auth" + "yunion.io/x/onecloud/pkg/util/huawei/client/responses" +) + +/* +https://support.huaweicloud.com/api-oce/zh-cn_topic_0075195195.html +客户运营能力API的Endpoint为“bss.cn-north-1.myhuaweicloud.com”。该Endpoint为全局Endpoint，中国站所有区域均可使用。 +如何获取合作伙伴ID https://support.huaweicloud.com/bpconsole_faq/zh-cn_topic_0081005893.html +注意事项： +客户查询自身的账户余额的时候，只允许使用客户自身的AK/SK或者Token调用。 +*/ +type SBalanceManager struct { + domainId string // 租户ID + ResourceManager +} + +type balanceCtx struct { + domainId string +} + +// https://support.huaweicloud.com/api-bpconsole/zh-cn_topic_0075213309.html +// 这个manager非常特殊。url hardcode +func (self *balanceCtx) GetPath() string { + return fmt.Sprintf("%s/customer/account-mgr", self.domainId) +} + +// 这个manager非常特殊。只有List 和 SetDomainId方法可用。其他方法未验证 +func NewBalanceManager(signer auth.Signer) *SBalanceManager { + return &SBalanceManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameBSS, + Region: "cn-north-1", + ProjectId: "", + version: "v1.0", + Keyword: "account_balance", + KeywordPlural: "account_balances", + + ResourceKeyword: "balances", + }} +} + +func (self *SBalanceManager) List(querys map[string]string) (*responses.ListResult, error) { + if len(self.domainId) == 0 { + return nil, fmt.Errorf("domainId is emtpy.Use SetDomainId method to set.") + } + + ctx := &balanceCtx{domainId: self.domainId} + return self.ListInContext(ctx, querys) +} + +func (self *SBalanceManager) SetDomainId(domainId string) { + self.domainId = domainId +} diff --git a/pkg/util/huawei/client/modules/mod_disks.go b/pkg/util/huawei/client/modules/mod_disks.go index e7c1dea171..5652cb56c3 100644 --- a/pkg/util/huawei/client/modules/mod_disks.go +++ b/pkg/util/huawei/client/modules/mod_disks.go @@ -26,5 +26,5 @@ func NewDiskManager(regionId string, projectId string, signer auth.Signer) *SDis } func (self *SDiskManager) List(querys map[string]string) (*responses.ListResult, error) { - return self.ListInContext(nil, "detail", querys) + return self.ListInContextWithSpec(nil, "detail", querys, self.KeywordPlural) } diff --git a/pkg/util/huawei/client/modules/mod_domains.go b/pkg/util/huawei/client/modules/mod_domains.go new file mode 100644 index 0000000000..244f8989f1 --- /dev/null +++ b/pkg/util/huawei/client/modules/mod_domains.go @@ -0,0 +1,25 @@ +package modules + +import ( + "net/http" + + "yunion.io/x/onecloud/pkg/util/huawei/client/auth" +) + +type SDomainManager struct { + ResourceManager +} + +func NewDomainManager(signer auth.Signer) *SDomainManager { + return &SDomainManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameIAM, + Region: "", + ProjectId: "", + version: "v3/auth", + Keyword: "domain", + KeywordPlural: "domains", + + ResourceKeyword: "domains", + }} +} diff --git a/pkg/util/huawei/client/modules/mod_flavors.go b/pkg/util/huawei/client/modules/mod_flavors.go new file mode 100644 index 0000000000..22bb6f1cab --- /dev/null +++ b/pkg/util/huawei/client/modules/mod_flavors.go @@ -0,0 +1,25 @@ +package modules + +import ( + "net/http" + + "yunion.io/x/onecloud/pkg/util/huawei/client/auth" +) + +type SFlavorManager struct { + ResourceManager +} + +func NewFlavorManager(regionId string, projectId string, signer auth.Signer) *SFlavorManager { + return &SFlavorManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameECS, + Region: regionId, + ProjectId: projectId, + version: "v1", + Keyword: "flavor", + KeywordPlural: "flavors", + + ResourceKeyword: "cloudservers/flavors", // 这个接口有点特殊，实际只用到了list一个方法。为了简便直接把cloudservers附上。 + }} +} diff --git a/pkg/util/huawei/client/modules/mod_images.go b/pkg/util/huawei/client/modules/mod_images.go index cd0970d3c9..0903ef0a3a 100644 --- a/pkg/util/huawei/client/modules/mod_images.go +++ b/pkg/util/huawei/client/modules/mod_images.go @@ -3,16 +3,34 @@ package modules import ( "net/http" + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/util/huawei/client/auth" + "yunion.io/x/onecloud/pkg/util/huawei/client/requests" ) type SImageManager struct { ResourceManager } -func NewImageManager(regionId string, signer auth.Signer) *SImageManager { +type imageProject struct { + projectId string +} + +// image创建接口若非默认project，需要在header中指定X-Project-ID。url中未携带project信息(与其他接口相比有一点特殊) +// 绕过了ResourceManager中的projectid。直接在发送json请求前注入X-Project-ID +func (self *imageProject) Process(request requests.IRequest) { + request.AddHeaderParam("X-Project-Id", self.projectId) +} + +func NewImageManager(regionId string, projectId string, signer auth.Signer) *SImageManager { + var requestHook imageProject + if len(projectId) > 0 { + requestHook = imageProject{projectId: projectId} + } + return &SImageManager{ResourceManager: ResourceManager{ - BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}, requestHook: &requestHook}, ServiceName: ServiceNameIMS, Region: regionId, ProjectId: "", @@ -23,3 +41,42 @@ func NewImageManager(regionId string, signer auth.Signer) *SImageManager { ResourceKeyword: "cloudimages", }} } + +func (self *SImageManager) Get(id string, querys map[string]string) (jsonutils.JSONObject, error) { + if querys == nil { + querys = make(map[string]string, 0) + } + + querys["id"] = id + // 这里默认使用private + if t, exists := querys["__imagetype"]; !exists || len(t) == 0 { + querys["__imagetype"] = "private" + } + + ret, err := self.ListInContext(nil, querys) + if err != nil { + return nil, err + } + + if ret.Data == nil || len(ret.Data) == 0 { + return nil, httperrors.NewNotFoundError("image %s not found", id) + } + + return ret.Data[0], nil +} + +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0020092108.html +// 删除image只能用这个manager +func NewOpenstackImageManager(regionId string, signer auth.Signer) *SImageManager { + return &SImageManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameIMS, + Region: regionId, + ProjectId: "", + version: "v2", + Keyword: "image", + KeywordPlural: "images", + + ResourceKeyword: "images", + }} +} diff --git a/pkg/util/huawei/client/modules/mod_jobs.go b/pkg/util/huawei/client/modules/mod_jobs.go new file mode 100644 index 0000000000..da9fb70a42 --- /dev/null +++ b/pkg/util/huawei/client/modules/mod_jobs.go @@ -0,0 +1,57 @@ +package modules + +import ( + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/util/huawei/client/auth" + "yunion.io/x/onecloud/pkg/util/huawei/client/responses" +) + +type SJobManager struct { + ResourceManager +} + +func NewJobManager(regionId string, projectId string, signer auth.Signer) *SJobManager { + return &SJobManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: "", + Region: regionId, + ProjectId: projectId, + version: "v1", + Keyword: "", + KeywordPlural: "", + + ResourceKeyword: "jobs", + }} +} + +func (self *SJobManager) Get(id string, querys map[string]string) (jsonutils.JSONObject, error) { + processedQuery, err := self.processQueryParam(querys) + if err != nil { + return nil, err + } + + return self.GetInContext(nil, id, processedQuery) +} + +func (self *SJobManager) List(querys map[string]string) (*responses.ListResult, error) { + processedQuery, err := self.processQueryParam(querys) + if err != nil { + return nil, err + } + return self.ListInContext(nil, processedQuery) +} + +// 兼容查询不同ServiceName服务的Job做的特殊处理。 +func (self *SJobManager) processQueryParam(querys map[string]string) (map[string]string, error) { + service_type, exists := querys["service_type"] + if !exists { + return querys, fmt.Errorf("must specific query parameter `service_type`. e.g. ecs|ims|iam") + } + + self.ServiceName = ServiceNameType(service_type) + delete(querys, "service_type") + return querys, nil +} diff --git a/pkg/util/huawei/client/modules/mod_orders.go b/pkg/util/huawei/client/modules/mod_orders.go new file mode 100644 index 0000000000..f542321634 --- /dev/null +++ b/pkg/util/huawei/client/modules/mod_orders.go @@ -0,0 +1,70 @@ +package modules + +import ( + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/util/huawei/client/auth" + "yunion.io/x/onecloud/pkg/util/huawei/client/manager" + "yunion.io/x/onecloud/pkg/util/huawei/client/responses" +) + +// domian 客户账号ID https://support.huaweicloud.com/oce_faq/zh-cn_topic_0113714840.html +type SOrderManager struct { + orderCtx manager.IManagerContext + ResourceManager +} + +type orderCtx struct { + domainId string +} + +// {domain_id}/common/ +// 这个manager非常特殊。url hardcode +func (self *orderCtx) GetPath() string { + return fmt.Sprintf("%s/common", self.domainId) +} + +// https://support.huaweicloud.com/api-oce/zh-cn_topic_0084961226.html +func NewOrderManager(regionId string, signer auth.Signer) *SOrderManager { + return &SOrderManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameBSS, + Region: regionId, + ProjectId: "", + version: "v1.0", + Keyword: "", + KeywordPlural: "", + + ResourceKeyword: "order-mgr", + }} +} + +func (self *SOrderManager) SetDomainId(domainId string) error { + if len(domainId) == 0 { + return fmt.Errorf("SetDomainId domain id should not be emtpy") + } + + self.orderCtx = &orderCtx{domainId: domainId} + return nil +} + +// 查询客户包周期资源列表 https://support.huaweicloud.com/api-oce/zh-cn_topic_0084961226.html +func (self *SOrderManager) List(querys map[string]string) (*responses.ListResult, error) { + if self.orderCtx == nil { + return nil, fmt.Errorf("domainId is emtpy.Use SetDomainId method to set.") + } + + return self.ListInContextWithSpec(self.orderCtx, "resources/detail", querys, "data") +} + +// 查询订单的资源开通详情 https://support.huaweicloud.com/api-oce/api_order_00001.html +func (self *SOrderManager) Get(id string, querys map[string]string) (jsonutils.JSONObject, error) { + if self.orderCtx == nil { + return nil, fmt.Errorf("domainId is emtpy.Use SetDomainId method to set.") + } + + // ！！！特殊调用 + return self.GetInContextWithSpec(self.orderCtx, "orders-resource", id, querys, "") +} diff --git a/pkg/util/huawei/client/modules/mod_port.go b/pkg/util/huawei/client/modules/mod_port.go index ee20decf08..d6a8188bff 100644 --- a/pkg/util/huawei/client/modules/mod_port.go +++ b/pkg/util/huawei/client/modules/mod_port.go @@ -4,15 +4,31 @@ import ( "net/http" "yunion.io/x/onecloud/pkg/util/huawei/client/auth" + "yunion.io/x/onecloud/pkg/util/huawei/client/requests" ) type SPortManager struct { ResourceManager } -func NewPortManager(regionId string, signer auth.Signer) *SPortManager { +type portProject struct { + projectId string +} + +// port接口查询时若非默认project，需要在header中指定X-Project-ID。url中未携带project信息(与其他接口相比有一点特殊) +// 绕过了ResourceManager中的projectid。直接在发送json请求前注入X-Project-ID +func (self *portProject) Process(request requests.IRequest) { + request.AddHeaderParam("X-Project-Id", self.projectId) +} + +func NewPortManager(regionId string, projectId string, signer auth.Signer) *SPortManager { + var requestHook portProject + if len(projectId) > 0 { + requestHook = portProject{projectId: projectId} + } + return &SPortManager{ResourceManager: ResourceManager{ - BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}, requestHook: &requestHook}, ServiceName: ServiceNameVPC, Region: regionId, ProjectId: "", diff --git a/pkg/util/huawei/client/modules/mod_secgroups.go b/pkg/util/huawei/client/modules/mod_secgroups.go index 03541270b2..dc51c81ffc 100644 --- a/pkg/util/huawei/client/modules/mod_secgroups.go +++ b/pkg/util/huawei/client/modules/mod_secgroups.go @@ -23,3 +23,17 @@ func NewSecurityGroupManager(regionId string, projectId string, signer auth.Sign ResourceKeyword: "security-groups", }} } + +func NewNovaSecurityGroupManager(regionId string, projectId string, signer auth.Signer) *SSecurityGroupManager { + return &SSecurityGroupManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameECS, + Region: regionId, + ProjectId: projectId, + version: "v2.1", + Keyword: "security_group", + KeywordPlural: "security_groups", + + ResourceKeyword: "os-security-groups", + }} +} diff --git a/pkg/util/huawei/client/modules/mod_servers.go b/pkg/util/huawei/client/modules/mod_servers.go index b39c2e9b06..aa508697b3 100644 --- a/pkg/util/huawei/client/modules/mod_servers.go +++ b/pkg/util/huawei/client/modules/mod_servers.go @@ -5,6 +5,7 @@ import ( "strconv" "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/util/huawei/client/auth" "yunion.io/x/onecloud/pkg/util/huawei/client/responses" ) @@ -13,6 +14,8 @@ type SServerManager struct { ResourceManager } +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212668.html +// v.1.1 新增支持创建包年/包月的弹性云服务器。！！但是不支持查询等调用 https://support.huaweicloud.com/api-ecs/zh-cn_topic_0093055772.html func NewServerManager(regionId, projectId string, signer auth.Signer) *SServerManager { return &SServerManager{ResourceManager: ResourceManager{ BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, @@ -39,5 +42,51 @@ func (self *SServerManager) List(querys map[string]string) (*responses.ListResul } querys["offset"] = strconv.Itoa(n + 1) } - return self.ListInContext(nil, "detail", querys) + return self.ListInContextWithSpec(nil, "detail", querys, self.KeywordPlural) +} + +/* +返回job id 或者 order id + +https://support.huaweicloud.com/api-ecs/zh-cn_topic_0093055772.html +创建按需的弹性云服务 ——> job_id 任务ID （返回数据uuid举例："70a599e0-31e7-49b7-b260-868f441e862b"） +包年包月机器 --> order_id (返回数据举例： "CS1711152257C60TL") +*/ +func (self *SServerManager) AsyncCreate(params jsonutils.JSONObject) (string, error) { + origin_version := self.version + self.version = "v1.1" + defer func() { self.version = origin_version }() + + ret, err := self.CreateInContextWithSpec(nil, "", params, "") + if err != nil { + return "", err + } + + // 按需机器 + jobId, err := ret.GetString("job_id") + if err == nil { + return jobId, nil + } + + // 包年包月机器 + return ret.GetString("order_id") +} + +func (self *SServerManager) Create(params jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return nil, fmt.Errorf("not supported.please use AsyncCreate") +} + +// 不推荐使用这个manager +func NewNovaServerManager(regionId, projectId string, signer auth.Signer) *SServerManager { + return &SServerManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameECS, + Region: regionId, + ProjectId: projectId, + version: "v2.1", + Keyword: "server", + KeywordPlural: "servers", + + ResourceKeyword: "servers", + }} } diff --git a/pkg/util/huawei/client/modules/mod_snapshots.go b/pkg/util/huawei/client/modules/mod_snapshots.go index ddf6fe5f57..3f76179c65 100644 --- a/pkg/util/huawei/client/modules/mod_snapshots.go +++ b/pkg/util/huawei/client/modules/mod_snapshots.go @@ -26,5 +26,22 @@ func NewSnapshotManager(regionId, projectId string, signer auth.Signer) *SSnapsh } func (self *SSnapshotManager) List(querys map[string]string) (*responses.ListResult, error) { - return self.ListInContext(nil, "detail", querys) + return self.ListInContextWithSpec(nil, "detail", querys, self.KeywordPlural) +} + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0051408629.html +// 回滚快照只能用这个manger。其他情况请不要使用 +// 另外，香港-亚太还支持另外一个接口。https://support.huaweicloud.com/api-evs/zh-cn_topic_0142374138.html +func NewOsSnapshotManager(regionId string, projectId string, signer auth.Signer) *SSnapshotManager { + return &SSnapshotManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameEVS, + Region: regionId, + ProjectId: projectId, + version: "v2", + Keyword: "snapshot", + KeywordPlural: "snapshots", + + ResourceKeyword: "os-vendor-snapshots", + }} } diff --git a/pkg/util/huawei/client/modules/mod_users.go b/pkg/util/huawei/client/modules/mod_users.go new file mode 100644 index 0000000000..15f2f0618d --- /dev/null +++ b/pkg/util/huawei/client/modules/mod_users.go @@ -0,0 +1,25 @@ +package modules + +import ( + "net/http" + + "yunion.io/x/onecloud/pkg/util/huawei/client/auth" +) + +type SUserManager struct { + ResourceManager +} + +func NewUserManager(signer auth.Signer) *SUserManager { + return &SUserManager{ResourceManager: ResourceManager{ + BaseManager: BaseManager{signer: signer, httpClient: &http.Client{}}, + ServiceName: ServiceNameIAM, + Region: "", + ProjectId: "", + version: "v3", + Keyword: "user", + KeywordPlural: "users", + + ResourceKeyword: "users", + }} +} diff --git a/pkg/util/huawei/consts.go b/pkg/util/huawei/consts.go index eb7e232c0a..ae0cc1ca0e 100644 --- a/pkg/util/huawei/consts.go +++ b/pkg/util/huawei/consts.go @@ -2,3 +2,21 @@ package huawei // 华为云返回的时间格式 const DATETIME_FORMAT = "2006-01-02T15:04:05.999999999" + +// Task status +const TASK_SUCCESS = "SUCCESS" + +// Charging Type +const ( + POST_PAID = "postPaid" // 按需付费 + PRE_PAID = "prePaid" // 包年包月 +) + +// 资源类型 https://support.huaweicloud.com/api-oce/zh-cn_topic_0079291752.html +const ( + RESOURCE_TYPE_VM = "hws.resource.type.vm" // ECS虚拟机 + RESOURCE_TYPE_VOLUME = "hws.resource.type.volume" // EVS卷 + RESOURCE_TYPE_BANDWIDTH = "hws.resource.type.bandwidth" // VPC带宽 + RESOURCE_TYPE_IP = "hws.resource.type.ip" // VPC公网IP + RESOURCE_TYPE_IMAGE = "hws.resource.type.marketplace" // 市场镜像 +) diff --git a/pkg/util/huawei/disk.go b/pkg/util/huawei/disk.go index e14395bc75..78257d8649 100644 --- a/pkg/util/huawei/disk.go +++ b/pkg/util/huawei/disk.go @@ -12,6 +12,20 @@ import ( "yunion.io/x/onecloud/pkg/compute/models" ) +/* +华为云云硬盘 +======创建========== +1.磁盘只能挂载到同一可用区的云服务器内，创建后不支持更换可用区 +2.计费模式 包年包月/按需计费 +3.*支持自动备份 + + +共享盘 和 普通盘：https://support.huaweicloud.com/productdesc-evs/zh-cn_topic_0032860759.html +根据是否支持挂载至多台云服务器可以将云硬盘分为非共享云硬盘和共享云硬盘。 +一个非共享云硬盘只能挂载至一台云服务器，而一个共享云硬盘可以同时挂载至多台云服务器。 +单个共享云硬盘最多可同时挂载给16个云服务器。目前，共享云硬盘只适用于数据盘，不支持系统盘。 +*/ + type Attachment struct { ServerID string `json:"server_id"` AttachmentID string `json:"attachment_id"` @@ -41,7 +55,7 @@ type VolumeImageMetadata struct { MinDisk string `json:"min_disk"` SupportKVM string `json:"__support_kvm"` VirtualEnvType string `json:"virtual_env_type"` - Size string `json:"size"` + SizeGB string `json:"size"` OSVersion string `json:"__os_version"` OSBit string `json:"__os_bit"` SupportKVMHi1822Hiovs string `json:"__support_kvm_hi1822_hiovs"` @@ -65,7 +79,7 @@ type SDisk struct { Status string `json:"status"` Attachments []Attachment `json:"attachments"` Description string `json:"description"` - Size int64 `json:"size"` + SizeGB int `json:"size"` Metadata DiskMeta `json:"metadata"` Encrypted bool `json:"encrypted"` Bootable string `json:"bootable"` @@ -182,7 +196,7 @@ func (self *SDisk) GetDiskFormat() string { } func (self *SDisk) GetDiskSizeMB() int { - return int(self.Size * 1024) + return int(self.SizeGB * 1024) } func (self *SDisk) checkAutoDelete(attachments []Attachment) bool { @@ -259,11 +273,26 @@ func (self *SDisk) GetAccessPath() string { } func (self *SDisk) Delete(ctx context.Context) error { - panic("implement me") + if _, err := self.storage.zone.region.GetDisk(self.GetId()); err == cloudprovider.ErrNotFound { + log.Errorf("Failed to find disk %s when delete", self.GetId()) + return nil + } + return self.storage.zone.region.DeleteDisk(self.GetId()) } func (self *SDisk) CreateISnapshot(ctx context.Context, name string, desc string) (cloudprovider.ICloudSnapshot, error) { - panic("implement me") + if snapshotId, err := self.storage.zone.region.CreateSnapshot(self.GetId(), name, desc); err != nil { + log.Errorf("createSnapshot fail %s", err) + return nil, err + } else if snapshot, err := self.getSnapshot(snapshotId); err != nil { + return nil, err + } else { + snapshot.region = self.storage.zone.region + if err := cloudprovider.WaitStatus(snapshot, models.SNAPSHOT_READY, 15*time.Second, 3600*time.Second); err != nil { + return nil, err + } + return snapshot, nil + } } func (self *SDisk) getSnapshot(snapshotId string) (*SSnapshot, error) { @@ -302,15 +331,17 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { } func (self *SDisk) Resize(ctx context.Context, newSizeMB int64) error { - panic("implement me") + sizeGb := newSizeMB / 1024 + return self.storage.zone.region.resizeDisk(self.GetId(), sizeGb) } func (self *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) { - panic("implement me") + return self.storage.zone.region.resetDisk(self.GetId(), snapshotId) } +// 华为云不支持重置 func (self *SDisk) Rebuild(ctx context.Context) error { - panic("implement me") + return cloudprovider.ErrNotSupported } func (self *SRegion) GetDisk(diskId string) (*SDisk, error) { @@ -332,3 +363,56 @@ func (self *SRegion) GetDisks(zoneId string, offset int, limit int) ([]SDisk, in err := DoList(self.ecsClient.Disks.List, querys, &disks) return disks, len(disks), err } + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0058762427.html +func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, snapshotId string, desc string) (string, error) { + params := jsonutils.NewDict() + volumeObj := jsonutils.NewDict() + volumeObj.Add(jsonutils.NewString(name), "name") + volumeObj.Add(jsonutils.NewString(zoneId), "availability_zone") + volumeObj.Add(jsonutils.NewString(desc), "description") + volumeObj.Add(jsonutils.NewString(category), "volume_type") + volumeObj.Add(jsonutils.NewInt(int64(sizeGb)), "size") + if len(snapshotId) > 0 { + volumeObj.Add(jsonutils.NewString(snapshotId), "snapshot_id") + } + + params.Add(volumeObj, "volume") + + disk := SDisk{} + err := DoCreate(self.ecsClient.Disks.Create, params, &disk) + return disk.ID, err +} + +func (self *SRegion) DeleteDisk(diskId string) error { + return DoDelete(self.ecsClient.Disks.Delete, diskId, nil, nil) +} + +/* +扩容状态为available的云硬盘时，没有约束限制。 +扩容状态为in-use的云硬盘时，有以下约束： +不支持共享云硬盘，即multiattach参数值必须为false。 +云硬盘所挂载的云服务器状态必须为ACTIVE、PAUSED、SUSPENDED、SHUTOFF才支持扩容 +*/ +func (self *SRegion) resizeDisk(diskId string, sizeGB int64) error { + params := jsonutils.NewDict() + osExtendObj := jsonutils.NewDict() + osExtendObj.Add(jsonutils.NewInt(sizeGB), "new_size") // GB + params.Add(osExtendObj, "os-extend") + _, err := self.ecsClient.Disks.PerformAction2("action", diskId, params, "") + return err +} + +/* +https://support.huaweicloud.com/api-evs/zh-cn_topic_0051408629.html +只支持快照回滚到源云硬盘，不支持快照回滚到其它指定云硬盘。 +只有云硬盘状态处于“available”或“error_rollbacking”状态才允许快照回滚到源云硬盘。 +*/ +func (self *SRegion) resetDisk(diskId, snapshotId string) (string, error) { + params := jsonutils.NewDict() + rollbackObj := jsonutils.NewDict() + rollbackObj.Add(jsonutils.NewString(diskId), "volume_id") + params.Add(rollbackObj, "rollback") + _, err := self.ecsClient.OsSnapshots.PerformAction2("rollback", snapshotId, params, "") + return diskId, err +} diff --git a/pkg/util/huawei/domain.go b/pkg/util/huawei/domain.go new file mode 100644 index 0000000000..cc9eb3e0e2 --- /dev/null +++ b/pkg/util/huawei/domain.go @@ -0,0 +1,38 @@ +package huawei + +import "yunion.io/x/onecloud/pkg/util/huawei/client" + +// https://support.huaweicloud.com/api-iam/zh-cn_topic_0057845574.html +// 租户列表 +type SDomain struct { + Description string `json:"description"` + Enabled bool `json:"enabled"` + ID string `json:"id"` + Name string `json:"name"` +} + +func (self *SHuaweiClient) getDomains() ([]SDomain, error) { + huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.secret) + domains := make([]SDomain, 0) + err := DoList(huawei.Domains.List, nil, &domains) + return domains, err +} + +func (self *SHuaweiClient) getEnabledDomains() ([]SDomain, error) { + huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.secret) + domains := make([]SDomain, 0) + err := DoList(huawei.Domains.List, nil, &domains) + if err != nil { + return domains, err + } + + enabledDomains := make([]SDomain, 0) + for i := range domains { + domain := domains[i] + if domain.Enabled { + enabledDomains = append(enabledDomains, domain) + } + } + + return enabledDomains, err +} diff --git a/pkg/util/huawei/eip.go b/pkg/util/huawei/eip.go index 3334245e74..32d86fa6ac 100644 --- a/pkg/util/huawei/eip.go +++ b/pkg/util/huawei/eip.go @@ -1,6 +1,7 @@ package huawei import ( + "fmt" "time" "yunion.io/x/jsonutils" @@ -11,8 +12,8 @@ import ( type TInternetChargeType string const ( - InternetChargeByTraffic = TInternetChargeType("PayByTraffic") - InternetChargeByBandwidth = TInternetChargeType("PayByBandwidth") + InternetChargeByTraffic = TInternetChargeType("traffic") + InternetChargeByBandwidth = TInternetChargeType("bandwidth") ) type Port struct { @@ -143,7 +144,6 @@ func (self *SEipAddress) GetAssociationType() string { } func (self *SEipAddress) GetAssociationExternalId() string { - // todo： implement me返回关联的实例 // network/0273a359d61847fc83405926c958c746/ext-floatingips?tenantId=0273a359d61847fc83405926c958c746&limit=2000 // 只能通过 port id 反查device id. if len(self.PortId) > 0 { @@ -185,17 +185,35 @@ func (self *SEipAddress) Delete() error { } func (self *SEipAddress) Associate(instanceId string) error { - err := self.region.AssociateEip(self.ID, instanceId) + portId, err := self.region.GetInstancePortId(instanceId) if err != nil { return err } + + if len(self.PortId) > 0 { + if self.PortId == portId { + return nil + } + + return fmt.Errorf("eip %s aready associate with port %s", self.GetId(), self.PortId) + } + + err = self.region.AssociateEipWithPortId(self.ID, portId) + if err != nil { + return err + } + err = cloudprovider.WaitStatus(self, models.EIP_STATUS_READY, 10*time.Second, 180*time.Second) return err } func (self *SEipAddress) Dissociate() error { - // todo : implement me - err := self.region.DissociateEip(self.ID, "") + port, err := self.region.GetPort(self.PortId) + if err != nil { + return err + } + + err = self.region.DissociateEip(self.ID, port.DeviceID) if err != nil { return err } @@ -204,13 +222,48 @@ func (self *SEipAddress) Dissociate() error { } func (self *SEipAddress) ChangeBandwidth(bw int) error { - return self.region.UpdateEipBandwidth(self.ID, bw) + return self.region.UpdateEipBandwidth(self.BandwidthID, bw) +} + +func (self *SRegion) GetInstancePortId(instanceId string) (string, error) { + // 目前只绑定一个网卡 + // todo: 还需要按照ports状态进行过滤 + ports, err := self.GetPorts(instanceId) + if err != nil { + return "", err + } + + if len(ports) == 0 { + return "", fmt.Errorf("AssociateEip instance %s port is empty", instanceId) + } + + return ports[0].ID, nil } // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090596.html -func (region *SRegion) AllocateEIP(bwMbps int, chargeType TInternetChargeType) (*SEipAddress, error) { - // todo: implement me - return &SEipAddress{}, nil +func (self *SRegion) AllocateEIP(name string, bwMbps int, chargeType TInternetChargeType, bgpType string) (*SEipAddress, error) { + paramsStr := ` +{ + "publicip": { + "type": "%s", + "ip_version": 4 + }, + "bandwidth": { + "name": "%s", + "size": %d, + "share_type": "PER", + "charge_mode": "%s" + } +} +` + if len(bgpType) == 0 { + return nil, fmt.Errorf("AllocateEIP bgp type should not be empty") + } + paramsStr = fmt.Sprintf(paramsStr, bgpType, name, bwMbps, chargeType) + params, _ := jsonutils.ParseString(paramsStr) + eip := SEipAddress{} + err := DoCreate(self.ecsClient.Eips.Create, params, &eip) + return &eip, err } func (self *SRegion) GetEip(eipId string) (*SEipAddress, error) { @@ -221,23 +274,62 @@ func (self *SRegion) GetEip(eipId string) (*SEipAddress, error) { } func (self *SRegion) DeallocateEIP(eipId string) error { - // todo : implement me - return cloudprovider.ErrNotSupported + _, err := self.ecsClient.Eips.Delete(eipId, nil) + return err } func (self *SRegion) AssociateEip(eipId string, instanceId string) error { - // todo : implement me - return cloudprovider.ErrNotSupported + portId, err := self.GetInstancePortId(instanceId) + if err != nil { + return err + } + return self.AssociateEipWithPortId(eipId, portId) +} + +func (self *SRegion) AssociateEipWithPortId(eipId string, portId string) error { + params := jsonutils.NewDict() + publicIPObj := jsonutils.NewDict() + publicIPObj.Add(jsonutils.NewString(portId), "port_id") + params.Add(publicIPObj, "publicip") + + _, err := self.ecsClient.Eips.Update(eipId, params) + return err } func (self *SRegion) DissociateEip(eipId string, instanceId string) error { - // todo : implement me - return cloudprovider.ErrNotSupported + eip, err := self.GetEip(eipId) + if err != nil { + return err + } + + // 已经是解绑状态 + if eip.Status == "DOWN" { + return nil + } + + remoteInstanceId := eip.GetAssociationExternalId() + if remoteInstanceId != instanceId { + return fmt.Errorf("eip %s associate with another instance %s", eipId, remoteInstanceId) + } + + paramsStr := `{"publicip":{"port_id":null}}` + params, _ := jsonutils.ParseString(paramsStr) + _, err = self.ecsClient.Eips.Update(eipId, params) + return err } -func (self *SRegion) UpdateEipBandwidth(eipId string, bw int) error { - // todo : implement me - return cloudprovider.ErrNotSupported +func (self *SRegion) UpdateEipBandwidth(bandwidthId string, bw int) error { + paramStr := `{ + "bandwidth": + { + "size": %d + } + }` + + paramStr = fmt.Sprintf(paramStr, bw) + params, _ := jsonutils.ParseString(paramStr) + _, err := self.ecsClient.Bandwidths.Update(bandwidthId, params) + return err } func (self *SRegion) GetEipBandwidth(bandwidthId string) (Bandwidth, error) { @@ -248,6 +340,18 @@ func (self *SRegion) GetEipBandwidth(bandwidthId string) (Bandwidth, error) { func (self *SRegion) GetPort(portId string) (Port, error) { port := Port{} - err := DoGet(self.ecsClient.Bandwidths.Get, portId, nil, &port) + err := DoGet(self.ecsClient.Port.Get, portId, nil, &port) return port, err } + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0030591299.html +func (self *SRegion) GetPorts(instanceId string) ([]Port, error) { + ports := make([]Port, 0) + querys := map[string]string{} + if len(instanceId) > 0 { + querys["device_id"] = instanceId + } + + err := DoList(self.ecsClient.Port.List, querys, &ports) + return ports, err +} diff --git a/pkg/util/huawei/host.go b/pkg/util/huawei/host.go index 7acbed9ae7..ab9d7d3222 100644 --- a/pkg/util/huawei/host.go +++ b/pkg/util/huawei/host.go @@ -4,6 +4,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" @@ -53,7 +54,10 @@ func (self *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) { return nil, err } - vms = append(vms, parts...) + for _, vm := range parts { + vm.host = self + vms = append(vms, vm) + } if count < limit { break @@ -80,6 +84,7 @@ func (self *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) { func (self *SHost) GetIVMById(id string) (cloudprovider.ICloudVM, error) { vm, err := self.zone.region.GetInstanceByID(id) + vm.host = self return &vm, err } @@ -165,18 +170,138 @@ func (self *SHost) GetManagerId() string { return self.zone.region.client.providerId } -func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, vswitchId string, ipAddr string, desc string, - passwd string, storageType string, diskSizes []int, publicKey string, extSecGrpId string, userData string, billingCycle *billing.SBillingCycle) (cloudprovider.ICloudVM, error) { - // todo: implement me - panic("implement me") +func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) { + instance, err := self.zone.region.GetInstanceByID(instanceId) + if err != nil { + return nil, err + } + + instance.host = self + return &instance, nil } -func (self *SHost) CreateVM2(name string, imgId string, sysDiskSize int, instanceType string, vswitchId string, ipAddr string, desc string, - passwd string, storageType string, diskSizes []int, publicKey string, extSecGrpId string, userData string, billingCycle *billing.SBillingCycle) (cloudprovider.ICloudVM, error) { - // todo: implement me - panic("implement me") +func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, networkId string, ipAddr string, desc string, + passwd string, storageType string, diskSizes []int, publicKey string, extSecGrpId string, userData string, bc *billing.SBillingCycle) (cloudprovider.ICloudVM, error) { + vmId, err := self._createVM(name, imgId, sysDiskSize, cpu, memMB, "", networkId, ipAddr, desc, passwd, storageType, diskSizes, publicKey, extSecGrpId, userData, bc) + if err != nil { + return nil, err + } + + vm, err := self.GetInstanceById(vmId) + if err != nil { + return nil, err + } + + return vm, err +} + +func (self *SHost) CreateVM2(name string, imgId string, sysDiskSize int, instanceType string, networkId string, ipAddr string, desc string, + passwd string, storageType string, diskSizes []int, publicKey string, extSecGrpId string, userData string, bc *billing.SBillingCycle) (cloudprovider.ICloudVM, error) { + vmId, err := self._createVM(name, imgId, sysDiskSize, 0, 0, instanceType, networkId, ipAddr, desc, passwd, storageType, diskSizes, publicKey, extSecGrpId, userData, bc) + if err != nil { + return nil, err + } + + vm, err := self.GetInstanceById(vmId) + if err != nil { + return nil, err + } + + return vm, err } func (self *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { return nil, cloudprovider.ErrNotSupported } + +func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, instanceType string, + networkId string, ipAddr string, desc string, passwd string, + storageType string, diskSizes []int, publicKey string, secgroupId string, + userData string, bc *billing.SBillingCycle) (string, error) { + net := self.zone.getNetworkById(networkId) + if net == nil { + return "", fmt.Errorf("invalid network ID %s", networkId) + } + + if net.wire == nil { + log.Errorf("network's wire is empty") + return "", fmt.Errorf("network's wire is empty") + } + + if net.wire.vpc == nil { + log.Errorf("wire's vpc is empty") + return "", fmt.Errorf("wire's vpc is empty") + } + + // 同步keypair + var err error + keypair := "" + if len(publicKey) > 0 { + keypair, err = self.zone.region.syncKeypair(publicKey) + if err != nil { + return "", err + } + } + + // 镜像及硬盘配置 + img, err := self.zone.region.GetImage(imgId) + if err != nil { + log.Errorf("getiamge %s fail %s", imgId, err) + return "", err + } + if img.Status != ImageStatusActive { + log.Errorf("image %s status %s", imgId, img.Status) + return "", fmt.Errorf("image not ready") + } + + disks := make([]SDisk, len(diskSizes)+1) + disks[0].SizeGB = img.SizeGB + if sysDiskSize > 0 && sysDiskSize > img.SizeGB { + disks[0].SizeGB = sysDiskSize + } + disks[0].VolumeType = storageType + + for i, sz := range diskSizes { + disks[i+1].SizeGB = sz + disks[i+1].VolumeType = storageType + } + + secgroup, err := self.zone.region.GetSecurityGroupDetails(secgroupId) + if err != nil { + return "", err + } + + // 创建实例 + if len(instanceType) > 0 { + log.Debugf("Try instancetype : %s", instanceType) + vmId, err := self.zone.region.CreateInstance(name, imgId, instanceType, networkId, secgroupId, secgroup.VpcID, self.zone.GetId(), desc, disks, ipAddr, keypair, passwd, userData, bc) + if err != nil { + log.Errorf("Failed for %s: %s", instanceType, err) + return "", fmt.Errorf("Failed to create, specification %s not supported", instanceType) + } else { + return vmId, nil + } + } + + // 匹配实例类型 + instanceTypes, err := self.zone.region.GetMatchInstanceTypes(cpu, memMB, self.zone.GetId()) + if err != nil { + return "", err + } + if len(instanceTypes) == 0 { + return "", fmt.Errorf("instance type %dC%dMB not avaiable", cpu, memMB) + } + + for _, instType := range instanceTypes { + instanceTypeId := instType.Name + log.Debugf("Try instancetype : %s", instanceTypeId) + vmId, err := self.zone.region.CreateInstance(name, imgId, instanceType, networkId, secgroupId, secgroup.VpcID, self.zone.GetId(), desc, disks, ipAddr, keypair, passwd, userData, bc) + if err != nil { + log.Errorf("Failed for %s: %s", instanceTypeId, err) + } else { + return vmId, nil + } + } + + return "", fmt.Errorf("Failed to create, specification not supported") +} diff --git a/pkg/util/huawei/huawei.go b/pkg/util/huawei/huawei.go index d15456b340..4a00adacba 100644 --- a/pkg/util/huawei/huawei.go +++ b/pkg/util/huawei/huawei.go @@ -15,6 +15,7 @@ import ( 1.同步的子账户中有一条空记录.需要查原因 2.安全组同步需要进一步确认 3.实例接口需要进一步确认 +4.BGP type 目前是hard code在代码中。需要考虑从cloudmeta服务中查询 */ const ( @@ -220,13 +221,59 @@ func (self *SHuaweiClient) GetIStorageById(id string) (cloudprovider.ICloudStora return nil, cloudprovider.ErrNotFound } +// 总账户余额 type SAccountBalance struct { AvailableAmount float64 } +// 账户余额 +// https://support.huaweicloud.com/api-oce/zh-cn_topic_0109685133.html +type SBalance struct { + Amount float64 `json:"amount"` + Currency string `json:"currency"` + AccountID string `json:"account_id"` + AccountType int64 `json:"account_type"` + DesignatedAmount *int64 `json:"designated_amount,omitempty"` + CreditAmount *int64 `json:"credit_amount,omitempty"` + MeasureUnit int64 `json:"measure_unit"` +} + +// 这里的余额指的是所有租户的总余额 func (self *SHuaweiClient) QueryAccountBalance() (*SAccountBalance, error) { - // todo: implement me - return nil, nil + domains, err := self.getEnabledDomains() + if err != nil { + return nil, err + } + + amount := float64(0) + for _, domain := range domains { + v, err := self.queryDomainBalance(domain.ID) + if err != nil { + return nil, err + } + + amount += v + } + + return &SAccountBalance{AvailableAmount: amount}, nil +} + +// https://support.huaweicloud.com/api-bpconsole/zh-cn_topic_0075213309.html +func (self *SHuaweiClient) queryDomainBalance(domainId string) (float64, error) { + huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.secret) + huawei.Balances.SetDomainId(domainId) + balances := make([]SBalance, 0) + err := DoList(huawei.Balances.List, nil, &balances) + if err != nil { + return 0, err + } + + amount := float64(0) + for _, balance := range balances { + amount += balance.Amount + } + + return amount, nil } func (self *SHuaweiClient) GetVersion() string { diff --git a/pkg/util/huawei/image.go b/pkg/util/huawei/image.go index 944698ac93..f040a43bbe 100644 --- a/pkg/util/huawei/image.go +++ b/pkg/util/huawei/image.go @@ -2,16 +2,30 @@ package huawei import ( "context" + "fmt" + "strings" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) -type ImageStatusType string type ImageOwnerType string -const ImageOwnerSelf ImageOwnerType = "private" +const ( + ImageOwnerPublic ImageOwnerType = "gold" // 公共镜像：gold + ImageOwnerSelf ImageOwnerType = "private" // 私有镜像：private + ImageOwnerShared ImageOwnerType = "shared" // 共享镜像：shared +) + +const ( + ImageStatusQueued = "queued" // queued：表示镜像元数据已经创建成功，等待上传镜像文件。 + ImageStatusSaving = "saving" // saving：表示镜像正在上传文件到后端存储。 + ImageStatusDeleted = "deleted" // deleted：表示镜像已经删除。 + ImageStatusKilled = "killed" // killed：表示镜像上传错误。 + ImageStatusActive = "active" // active：表示镜像可以正常使用 +) // https://support.huaweicloud.com/api-ims/zh-cn_topic_0020091565.html type SImage struct { @@ -37,7 +51,7 @@ type SImage struct { Visibility string `json:"visibility"` VirtualEnvType string `json:"virtual_env_type"` Platform string `json:"__platform"` - Size int64 `json:"size"` + SizeGB int `json:"size"` OSBit string `json:"__os_bit"` OSVersion string `json:"__os_version"` Name string `json:"name"` @@ -60,11 +74,11 @@ func (self *SImage) GetGlobalId() string { func (self *SImage) GetStatus() string { switch self.Status { - case "queued": + case ImageStatusQueued: return models.IMAGE_STATUS_QUEUED - case "active": + case ImageStatusActive: return models.IMAGE_STATUS_ACTIVE - case "killed": + case ImageStatusKilled: return models.IMAGE_STATUS_KILLED default: return models.IMAGE_STATUS_KILLED @@ -101,7 +115,6 @@ func (self *SImage) GetMetadata() *jsonutils.JSONDict { } func (self *SImage) Delete(ctx context.Context) error { - // todo: implement me return self.storageCache.region.DeleteImage(self.GetId()) } @@ -139,6 +152,153 @@ func (self *SRegion) GetImages(status string, imagetype ImageOwnerType, name str } func (self *SRegion) DeleteImage(imageId string) error { - // todo: implement me - return nil + return DoDelete(self.ecsClient.OpenStackImages.Delete, imageId, nil, nil) +} + +func (self *SRegion) GetImageByName(name string) (*SImage, error) { + if len(name) == 0 { + return nil, fmt.Errorf("image name should not be empty") + } + + images, _, err := self.GetImages("", ImageOwnerType(""), name, 1, "") + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, cloudprovider.ErrNotFound + } + + log.Debugf("%d image found match name %s", len(images), name) + return &images[0], nil +} + +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0020092109.html +// os version 取值范围： https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html +// todo: 考虑使用镜像快速导入。 https://support.huaweicloud.com/api-ims/zh-cn_topic_0133188204.html +// 使用OBS文件创建镜像 +func (self *SRegion) ImportImageJob(name string, osDist string, osVersion string, osArch string, bucket string, key string, minDiskGB int64) (string, error) { + os_version, err := stdVersion(osDist, osVersion, osArch) + log.Debugf("%s %s %s: %s", osDist, osVersion, osArch, os_version) + if err != nil { + log.Debugf(err.Error()) + } + + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(name), "name") + image_url := fmt.Sprintf("%s:%s", bucket, key) + params.Add(jsonutils.NewString(image_url), "image_url") + if len(os_version) > 0 { + params.Add(jsonutils.NewString(os_version), "os_version") + } + params.Add(jsonutils.NewBool(true), "is_config_init") + params.Add(jsonutils.NewBool(true), "is_config") + params.Add(jsonutils.NewInt(minDiskGB), "min_disk") + + ret, err := self.ecsClient.Images.PerformAction2("action", "", params, "") + if err != nil { + return "", err + } + + return ret.GetString("job_id") +} + +func formatVersion(osDist string, osVersion string) (string, error) { + err := fmt.Errorf("unsupport version %s.reference: https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html", osVersion) + dist := strings.ToLower(osDist) + if dist == "ubuntu" || dist == "redhat" || dist == "centos" || dist == "oracle" || dist == "euleros" { + parts := strings.Split(osVersion, ".") + if len(parts) < 2 { + return "", err + } + + return parts[0] + "." + parts[1], nil + } + + if dist == "debian" { + parts := strings.Split(osVersion, ".") + if len(parts) < 3 { + return "", err + } + + return parts[0] + "." + parts[1] + "." + parts[2], nil + } + + if dist == "fedora" || dist == "windows" || dist == "suse" { + parts := strings.Split(osVersion, ".") + if len(parts) < 1 { + return "", err + } + + return parts[0], nil + } + + if dist == "opensuse" { + parts := strings.Split(osVersion, ".") + if len(parts) == 0 { + return "", err + } + + if len(parts) == 1 { + return parts[0], nil + } + + if len(parts) >= 2 { + return parts[0] + "." + parts[1], nil + } + } + + return "", err +} + +// todo: 如何保持同步更新 +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html +func stdVersion(osDist string, osVersion string, osArch string) (string, error) { + // 架构 + arch := "" + switch osArch { + case "64", "x86_64": + arch = "64bit" + case "32", "x86_32": + arch = "32bit" + default: + return "", fmt.Errorf("unsupported arch %s.reference: https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html", osArch) + } + + _dist := strings.Split(strings.TrimSpace(osDist), " ")[0] + _dist = strings.ToLower(_dist) + // 版本 + ver, err := formatVersion(_dist, osVersion) + if err != nil { + return "", err + } + + // 操作系统 + dist := "" + + switch _dist { + case "ubuntu": + return fmt.Sprintf("Ubuntu %s server %s", ver, arch), nil + case "redhat": + dist = "Redhat Linux Enterprise" + case "centos": + dist = "CentOS" + case "fedora": + dist = "Fedora" + case "debian": + dist = "Debian GNU/Linux" + case "windows": + dist = "Windows Server" + case "oracle": + dist = "Oracle Linux Server release" + case "suse": + dist = "SUSE Linux Enterprise Server" + case "opensuse": + dist = "OpenSUSE" + case "euleros": + dist = "EulerOS" + default: + return "", fmt.Errorf("unsupported os %s. reference: https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html", dist) + } + + return fmt.Sprintf("%s %s %s", dist, ver, arch), nil } diff --git a/pkg/util/huawei/instance.go b/pkg/util/huawei/instance.go index 1e9b10b14d..c2e471c013 100644 --- a/pkg/util/huawei/instance.go +++ b/pkg/util/huawei/instance.go @@ -2,18 +2,28 @@ package huawei import ( "context" + "fmt" "strings" "time" "strconv" + + "sort" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/huawei/client/modules" "yunion.io/x/pkg/util/osprofile" ) +const ( + InstanceStatusRunning = "ACTIVE" + InstanceStatusTerminated = "DELETED" + InstanceStatusStopped = "SHUTOFF" +) + type IpAddress struct { Version string `json:"version"` Addr string `json:"addr"` @@ -67,10 +77,10 @@ type SysTag struct { } // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0094148849.html +// https://support.huaweicloud.com/api-bpconsole/zh-cn_topic_0100166287.html v1.1 支持创建包年/包月的弹性云服务器 type SInstance struct { host *SHost - Fault interface{} `json:"fault"` ID string `json:"id"` Name string `json:"name"` Addresses map[string][]IpAddress `json:"addresses"` @@ -116,6 +126,36 @@ type SInstance struct { SecurityGroups []SecurityGroup `json:"security_groups"` } +func compareSet(currentSet []string, newSet []string) (add []string, remove []string, keep []string) { + sort.Strings(currentSet) + sort.Strings(newSet) + + i, j := 0, 0 + for i < len(currentSet) || j < len(newSet) { + if i < len(currentSet) && j < len(newSet) { + if currentSet[i] == newSet[j] { + keep = append(keep, currentSet[i]) + i += 1 + j += 1 + } else if currentSet[i] < newSet[j] { + remove = append(remove, currentSet[i]) + i += 1 + } else { + add = append(add, newSet[j]) + j += 1 + } + } else if i >= len(currentSet) { + add = append(add, newSet[j]) + j += 1 + } else if j >= len(newSet) { + remove = append(remove, currentSet[i]) + i += 1 + } + } + + return add, remove, keep +} + func (self *SInstance) GetId() string { return self.ID } @@ -171,11 +211,17 @@ func (self *SInstance) GetMetadata() *jsonutils.JSONDict { data.Update(meta) } } - secgroupIds := jsonutils.NewArray() - for _, secgroup := range self.SecurityGroups { - secgroupIds.Add(jsonutils.NewString(secgroup.Name)) + + _secIds, err := self.host.zone.region.GetInstanceSecrityGroupIds(self.GetId()) + if err != nil { + log.Errorf(err.Error()) + } else { + secgroupIds := jsonutils.NewArray() + for _, secId := range _secIds { + secgroupIds.Add(jsonutils.NewString(secId)) + } + data.Add(secgroupIds, "secgroupIds") } - data.Add(secgroupIds, "secgroupIds") return data } @@ -311,11 +357,22 @@ func (self *SInstance) GetMachine() string { } func (self *SInstance) AssignSecurityGroup(secgroupId string) error { - panic("implement me") + return self.AssignSecurityGroups([]string{secgroupId}) } func (self *SInstance) AssignSecurityGroups(secgroupIds []string) error { - panic("implement me") + currentSecgroups, err := self.host.zone.region.GetInstanceSecrityGroupIds(self.GetId()) + if err != nil { + return err + } + + add, remove, _ := compareSet(currentSecgroups, secgroupIds) + err = self.host.zone.region.assignSecurityGroups(add, self.GetId()) + if err != nil { + return err + } + + return self.host.zone.region.unassignSecurityGroups(remove, self.GetId()) } func (self *SInstance) GetHypervisor() string { @@ -323,59 +380,125 @@ func (self *SInstance) GetHypervisor() string { } func (self *SInstance) StartVM(ctx context.Context) error { - panic("implement me") + if self.Status == InstanceStatusRunning { + return nil + } + + timeout := 300 * time.Second + interval := 15 * time.Second + + startTime := time.Now() + for time.Now().Sub(startTime) < timeout { + err := self.Refresh() + if err != nil { + return err + } + + if self.GetStatus() == models.VM_RUNNING { + return nil + } else if self.GetStatus() == models.VM_READY { + err := self.host.zone.region.StartVM(self.GetId()) + if err != nil { + return err + } + } + time.Sleep(interval) + } + return cloudprovider.ErrTimeout } func (self *SInstance) StopVM(ctx context.Context, isForce bool) error { - panic("implement me") + if self.Status == InstanceStatusStopped { + return nil + } + + err := self.host.zone.region.StopVM(self.GetId(), isForce) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues } func (self *SInstance) DeleteVM(ctx context.Context) error { - panic("implement me") + for { + err := self.host.zone.region.DeleteVM(self.GetId()) + if err != nil && self.Status != InstanceStatusTerminated { + log.Errorf("DeleteVM fail: %s", err) + return err + } else { + break + } + } + + return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes } func (self *SInstance) UpdateVM(ctx context.Context, name string) error { - panic("implement me") + return self.host.zone.region.UpdateVM(self.GetId(), name) } +// https://support.huaweicloud.com/usermanual-ecs/zh-cn_topic_0032380449.html +// 创建云服务器过程中注入用户数据。支持注入文本、文本文件或gzip文件。 +// 注入内容，需要进行base64格式编码。注入内容（编码之前的内容）最大长度32KB。 +// 对于Linux弹性云服务器，adminPass参数传入时，user_data参数不生效。 func (self *SInstance) UpdateUserData(userData string) error { - panic("implement me") + return cloudprovider.ErrNotSupported } +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876349.html 使用原镜像重装 +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876971.html 更换系统盘操作系统 +// 不支持调整系统盘大小 +// todo: 支持注入user_data func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { - panic("implement me") + var err error + if self.Image.ID == imageId { + err = self.host.zone.region.RebuildRoot(ctx, self.GetId(), passwd, publicKey) + if err != nil { + return "", err + } + } else { + err = self.host.zone.region.ChangeRoot(ctx, self.GetId(), imageId, passwd, publicKey) + if err != nil { + return "", err + } + } + + // todo: wait job finished here + return "", nil } func (self *SInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { - panic("implement me") + return self.host.zone.region.DeployVM(self.GetId(), name, password, publicKey, deleteKeypair, description) } func (self *SInstance) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { - panic("implement me") + return self.host.zone.region.ChangeVMConfig(self.OSEXTAZAvailabilityZone, self.GetId(), ncpu, vmem, nil) } func (self *SInstance) ChangeConfig2(ctx context.Context, instanceType string) error { - panic("implement me") + return self.host.zone.region.ChangeVMConfig2(self.OSEXTAZAvailabilityZone, self.GetId(), instanceType, nil) } +// todo:// 返回jsonobject感觉很诡异。不能直接知道内部细节 func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { - panic("implement me") + return self.host.zone.region.GetInstanceVNCUrl(self.GetId()) } func (self *SInstance) AttachDisk(ctx context.Context, diskId string) error { - panic("implement me") + // todo: calc device + return self.host.zone.region.AttachDisk(self.GetId(), diskId, "") } func (self *SInstance) DetachDisk(ctx context.Context, diskId string) error { - panic("implement me") + return self.host.zone.region.DetachDisk(self.GetId(), diskId) } func (self *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { - panic("implement me") + return cloudprovider.ErrNotSupported } func (self *SInstance) Renew(bc billing.SBillingCycle) error { - panic("implement me") + return self.host.zone.region.RenewInstance(self.GetId(), bc) } func (self *SRegion) GetInstances(offset int, limit int) ([]SInstance, int, error) { @@ -410,3 +533,467 @@ func (self *SRegion) GetInstanceByIds(ids []string) ([]SInstance, int, error) { return instances, len(instances), nil } + +/* +系统盘大小取值范围：1-1024 GB，且必须不小于镜像min_disk. +*/ +type SServerCreate struct { + AvailabilityZone string `json:"availability_zone"` + Name string `json:"name"` + ImageRef string `json:"imageRef"` + RootVolume RootVolume `json:"root_volume"` + DataVolumes []DataVolume `json:"data_volumes"` + FlavorRef string `json:"flavorRef"` + UserData string `json:"user_data"` + Vpcid string `json:"vpcid"` + SecurityGroups []SecGroup `json:"security_groups"` + Nics []NIC `json:"nics"` + KeyName string `json:"key_name"` + AdminPass string `json:"adminPass"` + Count int64 `json:"count"` + Extendparam ServerExtendparam `json:"extendparam"` + ServerTags []ServerTag `json:"server_tags"` + Description string `json:"description"` +} + +type DataVolume struct { + Volumetype string `json:"volumetype"` + SizeGB int `json:"size"` + Extendparam *DataVolumeExtendparam `json:"extendparam,omitempty"` + Multiattach *bool `json:"multiattach,omitempty"` + HwPassthrough *string `json:"hw:passthrough,omitempty"` +} + +type DataVolumeExtendparam struct { + SnapshotID string `json:"snapshotId"` +} + +type ServerExtendparam struct { + ChargingMode string `json:"chargingMode"` // 计费模式 prePaid|postPaid + PeriodType string `json:"periodType"` // 周期类型：month|year + PeriodNum string `json:"periodNum"` // 订购周期数：periodType=month（周期类型为月）时，取值为[1，9]。periodType=year（周期类型为年）时，取值为1。 + IsAutoRenew string `json:"isAutoRenew"` // 是否自动续订 true|false + IsAutoPay string `json:"isAutoPay"` // 是否自动从客户的账户中支付 true|false + RegionID string `json:"regionID"` +} + +type NIC struct { + SubnetID string `json:"subnet_id"` // 网络ID. 与 SNetwork里的ID对应。统一使用这个ID + IpAddress string `json:"ip_address"` +} + +type RootVolume struct { + Volumetype string `json:"volumetype"` + SizeGB int `json:"size"` +} + +type SecGroup struct { + ID string `json:"id"` +} + +type ServerTag struct { + Key string `json:"key"` + Value string `json:"value"` +} + +/* +包月机器退订规则： https://support.huaweicloud.com/usermanual-billing/zh-cn_topic_0083138805.html +5天无理由全额退订：新购资源（不包含续费资源）在开通的五天内且退订次数不超过10次（每账号每年10次）的符合5天无理由全额退订。 +非5天无理由退订：不符合5天无理由全额退订条件的退订，都属于非5天无理由退订。非5天无理由退订，不限制退订次数，但需要收取退订手续费。 + +退订资源的方法： https://support.huaweicloud.com/usermanual-billing/zh-cn_topic_0072297197.html +*/ +func (self *SRegion) CreateInstance(name string, imageId string, instanceType string, SubnetId string, + securityGroupId string, vpcId string, zoneId string, desc string, disks []SDisk, ipAddr string, + keypair string, passwd string, userData string, bc *billing.SBillingCycle) (string, error) { + params := SServerCreate{} + params.AvailabilityZone = zoneId + params.Name = name + params.FlavorRef = instanceType + params.ImageRef = imageId + params.KeyName = keypair + params.AdminPass = passwd + params.UserData = userData + params.Description = desc + params.Count = 1 + params.Nics = []NIC{{SubnetID: SubnetId, IpAddress: ipAddr}} + params.SecurityGroups = []SecGroup{{ID: securityGroupId}} + params.Vpcid = vpcId + + for i, disk := range disks { + if i == 0 { + params.RootVolume.Volumetype = disk.VolumeType + params.RootVolume.SizeGB = disk.SizeGB + } else { + dataVolume := DataVolume{} + dataVolume.Volumetype = disk.VolumeType + dataVolume.SizeGB = disk.SizeGB + params.DataVolumes = append(params.DataVolumes, dataVolume) + } + } + + // billing type + if bc != nil { + params.Extendparam.ChargingMode = PRE_PAID + if bc.GetMonths() <= 9 { + params.Extendparam.PeriodNum = strconv.Itoa(bc.GetMonths()) + params.Extendparam.PeriodType = "month" + } else { + params.Extendparam.PeriodNum = strconv.Itoa(bc.GetYears()) + params.Extendparam.PeriodType = "year" + } + + params.Extendparam.RegionID = self.GetId() + params.Extendparam.IsAutoRenew = "false" + params.Extendparam.IsAutoPay = "true" + } else { + params.Extendparam.ChargingMode = POST_PAID + } + + serverObj := jsonutils.Marshal(params) + createParams := jsonutils.NewDict() + createParams.Add(serverObj, "server") + _id, err := self.ecsClient.Servers.AsyncCreate(createParams) + if err != nil { + return "", err + } + + // 按需计费 + var ids []string + if params.Extendparam.ChargingMode == POST_PAID { + ids, err = self.GetAllSubTaskEntityIDs(self.ecsClient.Servers.ServiceType(), _id, "server_id") + } else { + err = cloudprovider.WaitCreated(10*time.Second, 180*time.Second, func() bool { + ids, err = self.getAllResIdsByType(_id, RESOURCE_TYPE_VM) + if err != nil { + log.Debugf(err.Error()) + return false + } + + if len(ids) > 0 { + return true + } + + return false + }) + } + + if err != nil { + return "", err + } else if len(ids) == 0 { + return "", fmt.Errorf("CreateInstance job %s result is emtpy", _id) + } else if len(ids) == 1 { + return ids[0], nil + } else { + return "", fmt.Errorf("CreateInstance job %s mutliple instance id returned. %s", _id, ids) + } +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067161469.html +// 添加多个安全组时，建议最多为弹性云服务器添加5个安全组。 +// todo: 确认是否需要先删除，再进行添加操作 +func (self *SRegion) assignSecurityGroups(secgroupIds []string, instanceId string) error { + _, err := self.GetInstanceByID(instanceId) + if err != nil { + return err + } + + for i := range secgroupIds { + secId := secgroupIds[i] + params := jsonutils.NewDict() + secgroupObj := jsonutils.NewDict() + secgroupObj.Add(jsonutils.NewString(secId), "name") + params.Add(secgroupObj, "addSecurityGroup") + + _, err := self.ecsClient.NovaServers.PerformAction("action", instanceId, params) + if err != nil { + return err + } + } + return nil +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067161717.html +func (self *SRegion) unassignSecurityGroups(secgroupIds []string, instanceId string) error { + for i := range secgroupIds { + secId := secgroupIds[i] + params := jsonutils.NewDict() + secgroupObj := jsonutils.NewDict() + secgroupObj.Add(jsonutils.NewString(secId), "name") + params.Add(secgroupObj, "removeSecurityGroup") + + _, err := self.ecsClient.NovaServers.PerformAction("action", instanceId, params) + if err != nil { + return err + } + } + return nil +} + +func (self *SRegion) GetInstanceStatus(instanceId string) (string, error) { + instance, err := self.GetInstanceByID(instanceId) + if err != nil { + return "", err + } + return instance.Status, nil +} + +func (self *SRegion) instanceStatusChecking(instanceId, status string) error { + remoteStatus, err := self.GetInstanceStatus(instanceId) + if err != nil { + log.Errorf("Fail to get instance status: %s", err) + return err + } + if status != remoteStatus { + log.Errorf("instanceStatusChecking: vm status is %s expect %s", remoteStatus, status) + return cloudprovider.ErrInvalidStatus + } + + return nil +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212207.html +func (self *SRegion) StartVM(instanceId string) error { + if err := self.instanceStatusChecking(instanceId, InstanceStatusStopped); err != nil { + return err + } + + params := jsonutils.NewDict() + startObj := jsonutils.NewDict() + serversObj := jsonutils.NewArray() + serverObj := jsonutils.NewDict() + serverObj.Add(jsonutils.NewString(instanceId), "id") + serversObj.Add(serverObj) + startObj.Add(serversObj, "servers") + params.Add(startObj, "os-start") + _, err := self.ecsClient.Servers.PerformAction2("action", "", params, "") + return err +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212651.html +func (self *SRegion) StopVM(instanceId string, isForce bool) error { + if err := self.instanceStatusChecking(instanceId, InstanceStatusRunning); err != nil { + return err + } + + params := jsonutils.NewDict() + stopObj := jsonutils.NewDict() + serversObj := jsonutils.NewArray() + serverObj := jsonutils.NewDict() + serverObj.Add(jsonutils.NewString(instanceId), "id") + serversObj.Add(serverObj) + stopObj.Add(serversObj, "servers") + if isForce { + stopObj.Add(jsonutils.NewString("HARD"), "type") + } else { + stopObj.Add(jsonutils.NewString("SOFT"), "type") + } + params.Add(stopObj, "os-stop") + _, err := self.ecsClient.Servers.PerformAction2("action", "", params, "") + return err +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212679.html +// 只删除主机。弹性IP和数据盘需要单独删除 +func (self *SRegion) DeleteVM(instanceId string) error { + if err := self.instanceStatusChecking(instanceId, InstanceStatusStopped); err != nil { + return err + } + + params := jsonutils.NewDict() + serversObj := jsonutils.NewArray() + serverObj := jsonutils.NewDict() + serverObj.Add(jsonutils.NewString(instanceId), "id") + serversObj.Add(serverObj) + params.Add(serversObj, "servers") + params.Add(jsonutils.NewBool(false), "delete_publicip") + params.Add(jsonutils.NewBool(false), "delete_volume") + + _, err := self.ecsClient.Servers.PerformAction2("delete", "", params, "") + return err +} + +func (self *SRegion) UpdateVM(instanceId, name string) error { + params := jsonutils.NewDict() + serversObj := jsonutils.NewArray() + serverObj := jsonutils.NewDict() + serverObj.Add(jsonutils.NewString(instanceId), "id") + serversObj.Add(serverObj) + params.Add(serversObj, "servers") + params.Add(jsonutils.NewString(name), "name") + + _, err := self.ecsClient.Servers.PerformAction2("server-name", "", params, "") + return err +} + +func (self *SRegion) RebuildRoot(ctx context.Context, instanceId, passwd, publicKeyName string) error { + params := jsonutils.NewDict() + reinstallObj := jsonutils.NewDict() + // meta := jsonutils.NewDict() + // meta.Add(jsonutils.NewString(""), "user_data") + if len(passwd) > 0 { + reinstallObj.Add(jsonutils.NewString(passwd), "adminpass") + } else if len(publicKeyName) > 0 { + reinstallObj.Add(jsonutils.NewString(publicKeyName), "keyname") + } else { + return fmt.Errorf("both password and publicKey are empty.") + } + + params.Add(reinstallObj, "os-reinstall") + _, err := self.ecsClient.Servers.PerformAction2("reinstallos", instanceId, params, "") + return err +} + +func (self *SRegion) ChangeRoot(ctx context.Context, instanceId, imageId, passwd, publicKeyName string) error { + params := jsonutils.NewDict() + changeOsObj := jsonutils.NewDict() + // meta := jsonutils.NewDict() + // meta.Add(jsonutils.NewString(""), "user_data") + if len(passwd) > 0 { + changeOsObj.Add(jsonutils.NewString(passwd), "adminpass") + } else if len(publicKeyName) > 0 { + changeOsObj.Add(jsonutils.NewString(publicKeyName), "keyname") + } else { + return fmt.Errorf("both password and publicKey are empty.") + } + + changeOsObj.Add(jsonutils.NewString(imageId), "imageid") + params.Add(changeOsObj, "os-change") + + _, err := self.ecsClient.Servers.PerformAction2("changeos", instanceId, params, "") + return err +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212692.html +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0110109377.html +// 目前不支持直接重置密钥 +func (self *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, deleteKeypair bool, description string) error { + serverObj := jsonutils.NewDict() + if len(name) > 0 { + serverObj.Add(jsonutils.NewString(name), "name") + } + + if len(description) > 0 { + serverObj.Add(jsonutils.NewString(description), "description") + } + + if serverObj.Size() > 0 { + params := jsonutils.NewDict() + params.Add(serverObj, "server") + // 这里华为返回的image字段是字符串。和SInstance的定义的image是字典结构不一致。 + err := DoUpdate(self.ecsClient.NovaServers.Update, instanceId, params, nil) + if err != nil { + return err + } + } + + if len(password) > 0 { + params := jsonutils.NewDict() + passwdObj := jsonutils.NewDict() + passwdObj.Add(jsonutils.NewString(password), "new_password") + params.Add(passwdObj, "reset-password") + + err := DoUpdateWithSpec(self.ecsClient.NovaServers.UpdateInContextWithSpec, instanceId, "os-reset-password", params) + if err != nil { + return err + } + } + + return nil +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212653.html +func (self *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, vmem int, disks []*SDisk) error { + instanceTypes, err := self.GetMatchInstanceTypes(ncpu, vmem, zoneId) + if err != nil { + return err + } + + for _, t := range instanceTypes { + params := jsonutils.NewDict() + resizeObj := jsonutils.NewDict() + resizeObj.Add(jsonutils.NewString(t.ID), "flavorRef") + params.Add(resizeObj, "resize") + _, err := self.ecsClient.Servers.PerformAction2("resize", instanceId, params, "") + if err != nil { + log.Errorf("Failed for %s: %s", t.ID, err) + } else { + return nil + } + } + + return fmt.Errorf("Failed to change vm config, specification not supported") +} + +func (self *SRegion) ChangeVMConfig2(zoneId string, instanceId string, instanceType string, disks []*SDisk) error { + params := jsonutils.NewDict() + resizeObj := jsonutils.NewDict() + resizeObj.Add(jsonutils.NewString(instanceType), "flavorRef") + params.Add(resizeObj, "resize") + + _, err := self.ecsClient.Servers.PerformAction2("resize", instanceId, params, "") + return err +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0142763126.html +// 微版本2.6及以上? +func (self *SRegion) GetInstanceVNCUrl(instanceId string) (jsonutils.JSONObject, error) { + params := jsonutils.NewDict() + vncObj := jsonutils.NewDict() + vncObj.Add(jsonutils.NewString("novnc"), "type") + vncObj.Add(jsonutils.NewString("vnc"), "protocol") + params.Add(vncObj, "remote_console") + + ret, err := self.ecsClient.NovaServers.PerformAction2("remote-consoles", instanceId, params, "") + if err != nil { + return nil, err + } + + return ret, nil +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0022472987.html +// todo: 指定device +// XEN平台虚拟机device为必选参数。 +func (self *SRegion) AttachDisk(instanceId string, diskId string, device string) error { + params := jsonutils.NewDict() + volumeObj := jsonutils.NewDict() + volumeObj.Add(jsonutils.NewString(diskId), "volumeId") + params.Add(volumeObj, "volumeAttachment") + + _, err := self.ecsClient.Servers.PerformAction2("attachvolume", instanceId, params, "") + return err +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0022472988.html +// 默认非强制卸载。delete_flag=0 +func (self *SRegion) DetachDisk(instanceId string, diskId string) error { + path := fmt.Sprintf("%s/detachvolume/%s", instanceId, diskId) + return DoDelete(self.ecsClient.Servers.Delete, path, nil, nil) +} + +// 目前无接口支持 +func (region *SRegion) RenewInstance(instanceId string, bc billing.SBillingCycle) error { + return cloudprovider.ErrNotSupported +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0065817702.html +func (self *SRegion) GetInstanceSecrityGroupIds(instanceId string) ([]string, error) { + if len(instanceId) == 0 { + return nil, fmt.Errorf("GetInstanceSecrityGroups instanceId is empty") + } + + securitygroups := make([]SSecurityGroup, 0) + ctx := &modules.ManagerContext{InstanceManager: self.ecsClient.NovaServers, InstanceId: instanceId} + err := DoListInContext(self.ecsClient.NovaSecurityGroups.ListInContext, ctx, nil, &securitygroups) + if err != nil { + return nil, err + } + + securitygroupIds := []string{} + for _, secgroup := range securitygroups { + securitygroupIds = append(securitygroupIds, secgroup.GetId()) + } + + return securitygroupIds, nil +} diff --git a/pkg/util/huawei/instancenic.go b/pkg/util/huawei/instancenic.go index 9d5a1abded..20a34451d2 100644 --- a/pkg/util/huawei/instancenic.go +++ b/pkg/util/huawei/instancenic.go @@ -3,7 +3,7 @@ package huawei import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/util/huawei/client/manager" + "yunion.io/x/onecloud/pkg/util/huawei/client/modules" "yunion.io/x/pkg/util/netutils" ) @@ -11,13 +11,31 @@ import ( type Interface struct { PortState string `json:"port_state"` FixedIPS []FixedIP `json:"fixed_ips"` - NetID string `json:"net_id"` + NetID string `json:"net_id"` // 网络ID. 与 SNetwork里的ID对应。统一使用这个ID PortID string `json:"port_id"` MACAddr string `json:"mac_addr"` } +/* +subnet: {id: "b09877fc-90d4-4fc8-b343-e6e00cb2b233", name: "subnet-149c", cidr: "192.168.0.0/24",…} +availability_zone: "cn-north-1b" +cidr: "192.168.0.0/24" +dhcp_enable: true +dnsList: ["100.125.1.250", "100.125.21.250"] +gateway_ip: "192.168.0.1" +id: "b09877fc-90d4-4fc8-b343-e6e00cb2b233" +ipv6_enable: false +name: "subnet-149c" +neutron_network_id: "b09877fc-90d4-4fc8-b343-e6e00cb2b233" +neutron_subnet_id: "81fcfaa0-8e73-4472-9eba-3b2b7736d3a7" +primary_dns: "100.125.1.250" +secondary_dns: "100.125.21.250" +status: "ACTIVE" +tags: [] +vpc_id: "877f1feb-3dc8-4c2d-92e9-0d94fd7d79dd"} +*/ type FixedIP struct { - SubnetID string `json:"subnet_id"` + SubnetID string `json:"subnet_id"` // 子网ID, 与SNetwork中的 neutron_subnet_id对应. 注意!!! 并不是SNetwork ID。 IPAddress string `json:"ip_address"` } @@ -64,7 +82,7 @@ func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork { } func (self *SRegion) getSubnetIdsByInstanceId(instanceId string) ([]string, error) { - ctx := &manager.ManagerContext{InstanceManager: self.ecsClient.Servers, InstanceId: instanceId} + ctx := &modules.ManagerContext{InstanceManager: self.ecsClient.NovaServers, InstanceId: instanceId} interfaces := make([]Interface, 0) err := DoListInContext(self.ecsClient.Interface.ListInContext, ctx, nil, &interfaces) if err != nil { @@ -73,9 +91,7 @@ func (self *SRegion) getSubnetIdsByInstanceId(instanceId string) ([]string, erro subnets := make([]string, 0) for _, i := range interfaces { - for _, net := range i.FixedIPS { - subnets = append(subnets, net.SubnetID) - } + subnets = append(subnets, i.NetID) } return subnets, nil diff --git a/pkg/util/huawei/instancetype.go b/pkg/util/huawei/instancetype.go new file mode 100644 index 0000000000..fef332afd3 --- /dev/null +++ b/pkg/util/huawei/instancetype.go @@ -0,0 +1,44 @@ +package huawei + +import "strconv" + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212656.html +type SInstanceType struct { + ID string `json:"id"` + Name string `json:"name"` + Vcpus string `json:"vcpus"` + RamMB int `json:"ram"` // 内存大小 + OSExtraSpecs OSExtraSpecs `json:"os_extra_specs"` // 扩展规格 +} + +type OSExtraSpecs struct { + EcsPerformancetype string `json:"ecs:performancetype"` +} + +func (self *SRegion) fetchInstanceTypes(zoneId string) ([]SInstanceType, error) { + querys := map[string]string{} + if len(zoneId) > 0 { + querys["availability_zone"] = zoneId + } + + instanceTypes := make([]SInstanceType, 0) + err := DoList(self.ecsClient.Flavors.List, querys, &instanceTypes) + return instanceTypes, err +} + +func (self *SRegion) GetMatchInstanceTypes(cpu int, memMB int, zoneId string) ([]SInstanceType, error) { + instanceTypes, err := self.fetchInstanceTypes(zoneId) + if err != nil { + return nil, err + } + + ret := make([]SInstanceType, 0) + for _, t := range instanceTypes { + // cpu & mem & disk都匹配才行 + if t.Vcpus == strconv.Itoa(cpu) && t.RamMB == memMB { + ret = append(ret, t) + } + } + + return ret, nil +} diff --git a/pkg/util/huawei/keypair.go b/pkg/util/huawei/keypair.go index 47659d3aff..b32208292b 100644 --- a/pkg/util/huawei/keypair.go +++ b/pkg/util/huawei/keypair.go @@ -1,3 +1,96 @@ package huawei +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/aokoli/goutils" + "golang.org/x/crypto/ssh" + "yunion.io/x/jsonutils" +) + // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212676.html +type SKeypair struct { + Fingerprint string `json:"fingerprint"` + Name string `json:"name"` + PublicKey string `json:"public_key"` +} + +func (self *SRegion) getFingerprint(publicKey string) (string, error) { + pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey)) + if err != nil { + return "", fmt.Errorf("publicKey error %s", err) + } + + fingerprint := strings.Replace(ssh.FingerprintLegacyMD5(pk), ":", "", -1) + return fingerprint, nil +} + +func (self *SRegion) GetKeypairs() ([]SKeypair, int, error) { + keypairs := make([]SKeypair, 0) + err := DoList(self.ecsClient.Keypairs.List, nil, &keypairs) + return keypairs, len(keypairs), err +} + +func (self *SRegion) lookUpKeypair(publicKey string) (string, error) { + keypairs, _, err := self.GetKeypairs() + if err != nil { + return "", err + } + + fingerprint, err := self.getFingerprint(publicKey) + if err != nil { + return "", err + } + + for _, keypair := range keypairs { + if keypair.Fingerprint == fingerprint { + return keypair.Name, nil + } + } + + return "", fmt.Errorf("keypair not found %s", err) +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212678.html +func (self *SRegion) ImportKeypair(name, publicKey string) (*SKeypair, error) { + fingerprint, err := self.getFingerprint(publicKey) + if err != nil { + return nil, err + } + + keypair := SKeypair{ + Name: name, + PublicKey: publicKey, + Fingerprint: fingerprint, + } + + keypairObj := jsonutils.Marshal(keypair) + ret := SKeypair{} + err = DoCreate(self.ecsClient.Keypairs.Create, keypairObj, &ret) + return &ret, err +} + +func (self *SRegion) importKeypair(publicKey string) (string, error) { + prefix, e := goutils.RandomAlphabetic(6) + if e != nil { + return "", fmt.Errorf("publicKey error %s", e) + } + + name := prefix + strconv.FormatInt(time.Now().Unix(), 10) + if k, e := self.ImportKeypair(name, publicKey); e != nil { + return "", fmt.Errorf("keypair import error %s", e) + } else { + return k.Name, nil + } +} + +func (self *SRegion) syncKeypair(publicKey string) (string, error) { + name, e := self.lookUpKeypair(publicKey) + if e == nil { + return name, nil + } + return self.importKeypair(publicKey) +} diff --git a/pkg/util/huawei/latitud_and_longitude.go b/pkg/util/huawei/latitud_and_longitude.go index 80ec03bab7..e190e18898 100644 --- a/pkg/util/huawei/latitud_and_longitude.go +++ b/pkg/util/huawei/latitud_and_longitude.go @@ -2,4 +2,16 @@ package huawei import "yunion.io/x/onecloud/pkg/cloudprovider" -var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{} +// China: https://developer.huaweicloud.com/endpoint +// International: https://developer-intl.huaweicloud.com/endpoint +// ref: https://countrycode.org +var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{ + "cn-east-2": {Latitude: 31.210344, Longitude: 121.455364, City: "Shanghai", CountryCode: "CN"}, + "cn-north-1": {Latitude: 39.997743, Longitude: 116.304542, City: "Beijing", CountryCode: "CN"}, + "cn-south-1": {Latitude: 23.12911, Longitude: 113.264385, City: "Guangzhou", CountryCode: "CN"}, + "cn-south-2": {Latitude: 23.12911, Longitude: 113.264385, City: "Guangzhou", CountryCode: "CN"}, + "ap-southeast-1": {Latitude: 22.396428, Longitude: 114.109497, City: "HongKong", CountryCode: "CN"}, + "ap-southeast-2": {Latitude: 13.7563309, Longitude: 100.5017651, City: "Bangkok", CountryCode: "TH"}, + "eu-west-0": {Latitude: 48.856614, Longitude: 2.3522219, City: "Paris", CountryCode: "FR"}, + "cn-northeast-1": {Latitude: 38.91400300000001, Longitude: 121.614682, City: "Shanghai", CountryCode: "CN"}, +} diff --git a/pkg/util/huawei/network.go b/pkg/util/huawei/network.go index 87d5cd90e7..9090460fac 100644 --- a/pkg/util/huawei/network.go +++ b/pkg/util/huawei/network.go @@ -2,7 +2,7 @@ package huawei import ( "strconv" - "strings" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -50,13 +50,21 @@ func (self *SNetwork) GetGlobalId() string { return self.ID } +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090591.html func (self *SNetwork) GetStatus() string { - return strings.ToLower(self.Status) + switch self.Status { + case "ACTIVE", "UNKNOWN": + return models.NETWORK_STATUS_AVAILABLE // ? todo: // UNKNOWN + case "ERROR": + return models.NETWORK_STATUS_UNKNOWN + default: + return models.NETWORK_STATUS_UNKNOWN + } } func (self *SNetwork) Refresh() error { log.Debugf("network refresh %s", self.GetId()) - new, err := self.wire.zone.region.getNetwork(self.GetId()) + new, err := self.wire.region.getNetwork(self.GetId()) if err != nil { return err } @@ -79,6 +87,7 @@ func (self *SNetwork) GetIpStart() string { pref, _ := netutils.NewIPV4Prefix(self.CIDR) startIp := pref.Address.NetAddr(pref.MaskLen) // 0 startIp = startIp.StepUp() // 1 + startIp = startIp.StepUp() // 2 return startIp.String() } @@ -112,8 +121,7 @@ func (self *SNetwork) GetIsPublic() bool { } func (self *SNetwork) Delete() error { - // todo: implement me - return nil + return self.wire.region.deleteNetwork(self.GetId()) } func (self *SNetwork) GetAllocTimeoutSeconds() int { @@ -141,3 +149,7 @@ func (self *SRegion) GetNetwroks(vpcId string, limit int, marker string) ([]SNet err := DoList(self.ecsClient.Subnets.List, querys, &networks) return networks, len(networks), err } + +func (self *SRegion) deleteNetwork(networkId string) error { + return DoDelete(self.ecsClient.Subnets.Delete, networkId, nil, nil) +} diff --git a/pkg/util/huawei/obs/auth.go b/pkg/util/huawei/obs/auth.go new file mode 100644 index 0000000000..82100c987f --- /dev/null +++ b/pkg/util/huawei/obs/auth.go @@ -0,0 +1,371 @@ +package obs + +import ( + "fmt" + "net/url" + "sort" + "strings" + "time" +) + +func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, expires int64) (requestUrl string, err error) { + + requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestUrl, err := url.Parse(requestUrl) + if err != nil { + return "", err + } + encodeHeaders(headers) + hostName := parsedRequestUrl.Host + + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) + + if obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { + if obsClient.conf.securityProvider.securityToken != "" { + params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + } + + if isV4 { + date, _ := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) + delete(headers, HEADER_DATE_CAMEL) + shortDate := date.Format(SHORT_DATE_FORMAT) + longDate := date.Format(LONG_DATE_FORMAT) + + signedHeaders, _headers := getSignedHeaders(headers) + + credential, scope := getCredential(obsClient.conf.securityProvider.ak, obsClient.conf.region, shortDate) + params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX + params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + params[PARAM_DATE_AMZ_CAMEL] = longDate + params[PARAM_EXPIRES_AMZ_CAMEL] = Int64ToString(expires) + params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = strings.Join(signedHeaders, ";") + + requestUrl, canonicalizedUrl = obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestUrl, _ = url.Parse(requestUrl) + stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, signedHeaders, _headers) + signature := getSignature(stringToSign, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate) + + requestUrl += fmt.Sprintf("&%s=%s", PARAM_SIGNATURE_AMZ_CAMEL, UrlEncode(signature, false)) + + } else { + originDate := headers[HEADER_DATE_CAMEL][0] + date, _ := time.Parse(RFC1123_FORMAT, originDate) + expires += date.Unix() + headers[HEADER_DATE_CAMEL] = []string{Int64ToString(expires)} + + stringToSign := getV2StringToSign(method, canonicalizedUrl, headers, obsClient.conf.signature == SignatureObs) + signature := UrlEncode(Base64Encode(HmacSha1([]byte(obsClient.conf.securityProvider.sk), []byte(stringToSign))), false) + if strings.Index(requestUrl, "?") < 0 { + requestUrl += "?" + } else { + requestUrl += "&" + } + delete(headers, HEADER_DATE_CAMEL) + requestUrl += fmt.Sprintf("AWSAccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(obsClient.conf.securityProvider.ak, false), + expires, signature) + } + } + + return +} + +func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, hostName string) (requestUrl string, err error) { + isObs := obsClient.conf.signature == SignatureObs + requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true) + parsedRequestUrl, err := url.Parse(requestUrl) + if err != nil { + return "", err + } + encodeHeaders(headers) + + if hostName == "" { + hostName = parsedRequestUrl.Host + } + + isV4 := obsClient.conf.signature == SignatureV4 + prepareHostAndDate(headers, hostName, isV4) + + if obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" { + doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") + } else { + if obsClient.conf.securityProvider.securityToken != "" { + headers[HEADER_STS_TOKEN_AMZ] = []string{obsClient.conf.securityProvider.securityToken} + } + ak := obsClient.conf.securityProvider.ak + sk := obsClient.conf.securityProvider.sk + var authorization string + if isV4 { + headers[HEADER_CONTENT_SHA256_AMZ] = []string{EMPTY_CONTENT_SHA256} + ret := v4Auth(ak, sk, obsClient.conf.region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, headers) + authorization = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else { + ret := v2Auth(ak, sk, method, canonicalizedUrl, headers, isObs) + hashPrefix := V2_HASH_PREFIX + if isObs { + hashPrefix = OBS_HASH_PREFIX + } + authorization = fmt.Sprintf("%s %s:%s", hashPrefix, ak, ret["Signature"]) + } + headers[HEADER_AUTH_CAMEL] = []string{authorization} + } + return +} + +func prepareHostAndDate(headers map[string][]string, hostName string, isV4 bool) { + headers[HEADER_HOST_CAMEL] = []string{hostName} + if date, ok := headers[HEADER_DATE_AMZ]; ok { + flag := false + if len(date) == 1 { + if isV4 { + if t, err := time.Parse(LONG_DATE_FORMAT, date[0]); err == nil { + headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(t)} + flag = true + } + } else { + if strings.HasSuffix(date[0], "GMT") { + headers[HEADER_DATE_CAMEL] = []string{date[0]} + flag = true + } + } + } + if !flag { + delete(headers, HEADER_DATE_AMZ) + } + } + if _, ok := headers[HEADER_DATE_CAMEL]; !ok { + headers[HEADER_DATE_CAMEL] = []string{FormatUtcToRfc1123(time.Now().UTC())} + } +} + +func encodeHeaders(headers map[string][]string) { + for key, values := range headers { + for index, value := range values { + values[index] = UrlEncode(value, true) + } + headers[key] = values + } +} + +func attachHeaders(headers map[string][]string, isObs bool) string { + length := len(headers) + _headers := make(map[string][]string, length) + keys := make([]string, 0, length) + + for key, value := range headers { + _key := strings.ToLower(strings.TrimSpace(key)) + if _key != "" { + prefixheader := HEADER_PREFIX + if isObs { + prefixheader = HEADER_PREFIX_OBS + } + if _key == "content-md5" || _key == "content-type" || _key == "date" || strings.HasPrefix(_key, prefixheader) { + keys = append(keys, _key) + _headers[_key] = value + } + } else { + delete(headers, key) + } + } + + for _, interestedHeader := range interested_headers { + if _, ok := _headers[interestedHeader]; !ok { + _headers[interestedHeader] = []string{""} + keys = append(keys, interestedHeader) + } + } + dateCamelHeader := PARAM_DATE_AMZ_CAMEL + dataHeader := HEADER_DATE_AMZ + if isObs { + dateCamelHeader = PARAM_DATE_OBS_CAMEL + dataHeader = HEADER_DATE_OBS + } + if _, ok := _headers[HEADER_DATE_CAMEL]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } else if _, ok := _headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + if _, ok := _headers[dataHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } else if _, ok := headers[dateCamelHeader]; ok { + _headers[HEADER_DATE_CAMEL] = []string{""} + } + } + + sort.Strings(keys) + + stringToSign := make([]string, 0, len(keys)) + for _, key := range keys { + var value string + prefixHeader := HEADER_PREFIX + prefixMetaHeader := HEADER_PREFIX_META + if isObs { + prefixHeader = HEADER_PREFIX_OBS + prefixMetaHeader = HEADER_PREFIX_META_OBS + } + if strings.HasPrefix(key, prefixHeader) { + if strings.HasPrefix(key, prefixMetaHeader) { + for index, v := range _headers[key] { + value += strings.TrimSpace(v) + if index != len(_headers[key])-1 { + value += "," + } + } + } else { + value = strings.Join(_headers[key], ",") + } + value = fmt.Sprintf("%s:%s", key, value) + } else { + value = strings.Join(_headers[key], ",") + } + stringToSign = append(stringToSign, value) + } + return strings.Join(stringToSign, "\n") +} + +func getV2StringToSign(method, canonicalizedUrl string, headers map[string][]string, isObs bool) string { + stringToSign := strings.Join([]string{method, "\n", attachHeaders(headers, isObs), "\n", canonicalizedUrl}, "") + doLog(LEVEL_DEBUG, "The v2 auth stringToSign:\n%s", stringToSign) + return stringToSign +} + +func v2Auth(ak, sk, method, canonicalizedUrl string, headers map[string][]string, isObs bool) map[string]string { + stringToSign := getV2StringToSign(method, canonicalizedUrl, headers, isObs) + return map[string]string{"Signature": Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign)))} +} + +func getScope(region, shortDate string) string { + return fmt.Sprintf("%s/%s/%s/%s", shortDate, region, V4_SERVICE_NAME, V4_SERVICE_SUFFIX) +} + +func getCredential(ak, region, shortDate string) (string, string) { + scope := getScope(region, shortDate) + return fmt.Sprintf("%s/%s", ak, scope), scope +} + +func getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, payload string, signedHeaders []string, headers map[string][]string) string { + canonicalRequest := make([]string, 0, 10+len(signedHeaders)*4) + canonicalRequest = append(canonicalRequest, method) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, canonicalizedUrl) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, queryUrl) + canonicalRequest = append(canonicalRequest, "\n") + + for _, signedHeader := range signedHeaders { + values, _ := headers[signedHeader] + for _, value := range values { + canonicalRequest = append(canonicalRequest, signedHeader) + canonicalRequest = append(canonicalRequest, ":") + canonicalRequest = append(canonicalRequest, value) + canonicalRequest = append(canonicalRequest, "\n") + } + } + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, strings.Join(signedHeaders, ";")) + canonicalRequest = append(canonicalRequest, "\n") + canonicalRequest = append(canonicalRequest, payload) + + _canonicalRequest := strings.Join(canonicalRequest, "") + doLog(LEVEL_DEBUG, "The v4 auth canonicalRequest:\n%s", _canonicalRequest) + + stringToSign := make([]string, 0, 7) + stringToSign = append(stringToSign, V4_HASH_PREFIX) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, longDate) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, scope) + stringToSign = append(stringToSign, "\n") + stringToSign = append(stringToSign, HexSha256([]byte(_canonicalRequest))) + + _stringToSign := strings.Join(stringToSign, "") + + doLog(LEVEL_DEBUG, "The v4 auth stringToSign:\n%s", _stringToSign) + return _stringToSign +} + +func getSignedHeaders(headers map[string][]string) ([]string, map[string][]string) { + length := len(headers) + _headers := make(map[string][]string, length) + signedHeaders := make([]string, 0, length) + for key, value := range headers { + _key := strings.ToLower(strings.TrimSpace(key)) + if _key != "" { + signedHeaders = append(signedHeaders, _key) + _headers[_key] = value + } else { + delete(headers, key) + } + } + sort.Strings(signedHeaders) + return signedHeaders, _headers +} + +func getSignature(stringToSign, sk, region, shortDate string) string { + key := HmacSha256([]byte(V4_HASH_PRE+sk), []byte(shortDate)) + key = HmacSha256(key, []byte(region)) + key = HmacSha256(key, []byte(V4_SERVICE_NAME)) + key = HmacSha256(key, []byte(V4_SERVICE_SUFFIX)) + return Hex(HmacSha256(key, []byte(stringToSign))) +} + +func V4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl string, headers map[string][]string) map[string]string { + return v4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl, headers) +} + +func v4Auth(ak, sk, region, method, canonicalizedUrl, queryUrl string, headers map[string][]string) map[string]string { + var t time.Time + if val, ok := headers[HEADER_DATE_AMZ]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[PARAM_DATE_AMZ_CAMEL]; ok { + var err error + t, err = time.Parse(LONG_DATE_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[HEADER_DATE_CAMEL]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else if val, ok := headers[strings.ToLower(HEADER_DATE_CAMEL)]; ok { + var err error + t, err = time.Parse(RFC1123_FORMAT, val[0]) + if err != nil { + t = time.Now().UTC() + } + } else { + t = time.Now().UTC() + } + shortDate := t.Format(SHORT_DATE_FORMAT) + longDate := t.Format(LONG_DATE_FORMAT) + + signedHeaders, _headers := getSignedHeaders(headers) + + credential, scope := getCredential(ak, region, shortDate) + + payload := EMPTY_CONTENT_SHA256 + if val, ok := headers[HEADER_CONTENT_SHA256_AMZ]; ok { + payload = val[0] + } + stringToSign := getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, payload, signedHeaders, _headers) + + signature := getSignature(stringToSign, sk, region, shortDate) + + ret := make(map[string]string, 3) + ret["Credential"] = credential + ret["SignedHeaders"] = strings.Join(signedHeaders, ";") + ret["Signature"] = signature + return ret +} diff --git a/pkg/util/huawei/obs/client.go b/pkg/util/huawei/obs/client.go new file mode 100644 index 0000000000..d8032974e4 --- /dev/null +++ b/pkg/util/huawei/obs/client.go @@ -0,0 +1,904 @@ +package obs + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "sort" + "strings" +) + +type ObsClient struct { + conf *config + httpClient *http.Client +} + +func New(ak, sk, endpoint string, configurers ...configurer) (*ObsClient, error) { + conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, endpoint: endpoint} + conf.maxRetryCount = -1 + for _, configurer := range configurers { + configurer(conf) + } + + if err := conf.initConfigWithDefault(); err != nil { + return nil, err + } + err := conf.getTransport() + if err != nil { + return nil, err + } + + if isWarnLogEnabled() { + info := make([]string, 3) + info[0] = fmt.Sprintf("[OBS SDK Version=%s", obs_sdk_version) + info[1] = fmt.Sprintf("Endpoint=%s", conf.endpoint) + accessMode := "Virtual Hosting" + if conf.pathStyle { + accessMode = "Path" + } + info[2] = fmt.Sprintf("Access Mode=%s]", accessMode) + doLog(LEVEL_WARN, strings.Join(info, "];[")) + } + doLog(LEVEL_DEBUG, "Create obsclient with config:\n%s\n", conf) + obsClient := &ObsClient{conf: conf, httpClient: &http.Client{Transport: conf.transport, CheckRedirect: checkRedirectFunc}} + return obsClient, nil +} + +func (obsClient ObsClient) Refresh(ak, sk, securityToken string) { + sp := &securityProvider{ak: strings.TrimSpace(ak), sk: strings.TrimSpace(sk), securityToken: strings.TrimSpace(securityToken)} + obsClient.conf.securityProvider = sp +} + +func (obsClient ObsClient) Close() { + obsClient.httpClient = nil + obsClient.conf.transport.CloseIdleConnections() + obsClient.conf = nil + SyncLog() +} + +func (obsClient ObsClient) ListBuckets(input *ListBucketsInput) (output *ListBucketsOutput, err error) { + if input == nil { + input = &ListBucketsInput{} + } + output = &ListBucketsOutput{} + err = obsClient.doActionWithoutBucket("ListBuckets", HTTP_GET, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) CreateBucket(input *CreateBucketInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("CreateBucketInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("CreateBucket", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucket(bucketName string) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucket", HTTP_DELETE, bucketName, defaultSerializable, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketStoragePolicy(input *SetBucketStoragePolicyInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketStoragePolicyInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketStoragePolicy", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} +func (obsClient ObsClient) getBucketStoragePolicyS3(bucketName string) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + var outputS3 *getBucketStoragePolicyOutputS3 + outputS3 = &getBucketStoragePolicyOutputS3{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStoragePolicy), outputS3) + if err != nil { + output = nil + } + output.BaseModel = outputS3.BaseModel + output.StorageClass = fmt.Sprintf("%s", outputS3.StorageClass) + return +} + +func (obsClient ObsClient) getBucketStoragePolicyObs(bucketName string) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + var outputObs *getBucketStoragePolicyOutputObs + outputObs = &getBucketStoragePolicyOutputObs{} + err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageClass), outputObs) + if err != nil { + output = nil + } + output.BaseModel = outputObs.BaseModel + output.StorageClass = outputObs.StorageClass + return +} +func (obsClient ObsClient) GetBucketStoragePolicy(bucketName string) (output *GetBucketStoragePolicyOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketStoragePolicyObs(bucketName) + } + return obsClient.getBucketStoragePolicyS3(bucketName) +} + +func (obsClient ObsClient) ListObjects(input *ListObjectsInput) (output *ListObjectsOutput, err error) { + if input == nil { + return nil, errors.New("ListObjectsInput is nil") + } + output = &ListObjectsOutput{} + err = obsClient.doActionWithBucket("ListObjects", HTTP_GET, input.Bucket, input, output) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +func (obsClient ObsClient) ListVersions(input *ListVersionsInput) (output *ListVersionsOutput, err error) { + if input == nil { + return nil, errors.New("ListVersionsInput is nil") + } + output = &ListVersionsOutput{} + err = obsClient.doActionWithBucket("ListVersions", HTTP_GET, input.Bucket, input, output) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +func (obsClient ObsClient) ListMultipartUploads(input *ListMultipartUploadsInput) (output *ListMultipartUploadsOutput, err error) { + if input == nil { + return nil, errors.New("ListMultipartUploadsInput is nil") + } + output = &ListMultipartUploadsOutput{} + err = obsClient.doActionWithBucket("ListMultipartUploads", HTTP_GET, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketQuota(input *SetBucketQuotaInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketQuotaInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketQuota", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketQuota(bucketName string) (output *GetBucketQuotaOutput, err error) { + output = &GetBucketQuotaOutput{} + err = obsClient.doActionWithBucket("GetBucketQuota", HTTP_GET, bucketName, newSubResourceSerial(SubResourceQuota), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) HeadBucket(bucketName string) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("HeadBucket", HTTP_HEAD, bucketName, defaultSerializable, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketMetadata(input *GetBucketMetadataInput) (output *GetBucketMetadataOutput, err error) { + output = &GetBucketMetadataOutput{} + err = obsClient.doActionWithBucket("GetBucketMetadata", HTTP_HEAD, input.Bucket, input, output) + if err != nil { + output = nil + } else { + ParseGetBucketMetadataOutput(output) + } + return +} + +func (obsClient ObsClient) GetBucketStorageInfo(bucketName string) (output *GetBucketStorageInfoOutput, err error) { + output = &GetBucketStorageInfoOutput{} + err = obsClient.doActionWithBucket("GetBucketStorageInfo", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageInfo), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) getBucketLocationS3(bucketName string) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + var outputS3 *getBucketLocationOutputS3 + outputS3 = &getBucketLocationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputS3) + if err != nil { + output = nil + } else { + output.BaseModel = outputS3.BaseModel + output.Location = outputS3.Location + } + return +} +func (obsClient ObsClient) getBucketLocationObs(bucketName string) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + var outputObs *getBucketLocationOutputObs + outputObs = &getBucketLocationOutputObs{} + err = obsClient.doActionWithBucket("GetBucketLocation", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLocation), outputObs) + if err != nil { + output = nil + } else { + output.BaseModel = outputObs.BaseModel + output.Location = outputObs.Location + } + return +} +func (obsClient ObsClient) GetBucketLocation(bucketName string) (output *GetBucketLocationOutput, err error) { + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketLocationObs(bucketName) + } + return obsClient.getBucketLocationS3(bucketName) +} + +func (obsClient ObsClient) SetBucketAcl(input *SetBucketAclInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketAclInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketAcl", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} +func (obsClient ObsClient) getBucketAclObs(bucketName string) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + var outputObs *getBucketAclOutputObs + outputObs = &getBucketAclOutputObs{} + err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), outputObs) + if err != nil { + output = nil + } else { + output.BaseModel = outputObs.BaseModel + for i, valGrant := range outputObs.Grants { + output.Grants[i].Delivered = valGrant.Delivered + output.Grants[i].Permission = valGrant.Permission + output.Grants[i].Grantee.DisplayName = valGrant.Grantee.DisplayName + output.Grants[i].Grantee.ID = valGrant.Grantee.ID + output.Grants[i].Grantee.Type = valGrant.Grantee.Type + output.Grants[i].Grantee.URI = GroupAllUsers + } + } + return +} +func (obsClient ObsClient) GetBucketAcl(bucketName string) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + if obsClient.conf.signature == SignatureObs { + return obsClient.getBucketAclObs(bucketName) + } + err = obsClient.doActionWithBucket("GetBucketAcl", HTTP_GET, bucketName, newSubResourceSerial(SubResourceAcl), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketPolicy(input *SetBucketPolicyInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketPolicy is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketPolicy", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketPolicy(bucketName string) (output *GetBucketPolicyOutput, err error) { + output = &GetBucketPolicyOutput{} + err = obsClient.doActionWithBucketV2("GetBucketPolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourcePolicy), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketPolicy(bucketName string) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketPolicy", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourcePolicy), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketCors(input *SetBucketCorsInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketCorsInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketCors", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketCors(bucketName string) (output *GetBucketCorsOutput, err error) { + output = &GetBucketCorsOutput{} + err = obsClient.doActionWithBucket("GetBucketCors", HTTP_GET, bucketName, newSubResourceSerial(SubResourceCors), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketCors(bucketName string) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketCors", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceCors), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketVersioning(input *SetBucketVersioningInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketVersioningInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketVersioning", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketVersioning(bucketName string) (output *GetBucketVersioningOutput, err error) { + output = &GetBucketVersioningOutput{} + err = obsClient.doActionWithBucket("GetBucketVersioning", HTTP_GET, bucketName, newSubResourceSerial(SubResourceVersioning), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketWebsiteConfiguration(input *SetBucketWebsiteConfigurationInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketWebsiteConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketWebsiteConfiguration", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketWebsiteConfiguration(bucketName string) (output *GetBucketWebsiteConfigurationOutput, err error) { + output = &GetBucketWebsiteConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketWebsiteConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceWebsite), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketWebsiteConfiguration(bucketName string) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketWebsiteConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceWebsite), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketLoggingConfiguration(input *SetBucketLoggingConfigurationInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketLoggingConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketLoggingConfiguration", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketLoggingConfiguration(bucketName string) (output *GetBucketLoggingConfigurationOutput, err error) { + output = &GetBucketLoggingConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketLoggingConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLogging), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketLifecycleConfiguration(input *SetBucketLifecycleConfigurationInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketLifecycleConfigurationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketLifecycleConfiguration", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketLifecycleConfiguration(bucketName string) (output *GetBucketLifecycleConfigurationOutput, err error) { + output = &GetBucketLifecycleConfigurationOutput{} + err = obsClient.doActionWithBucket("GetBucketLifecycleConfiguration", HTTP_GET, bucketName, newSubResourceSerial(SubResourceLifecycle), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketLifecycleConfiguration(bucketName string) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketLifecycleConfiguration", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceLifecycle), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketTagging(input *SetBucketTaggingInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketTaggingInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketTagging", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketTagging(bucketName string) (output *GetBucketTaggingOutput, err error) { + output = &GetBucketTaggingOutput{} + err = obsClient.doActionWithBucket("GetBucketTagging", HTTP_GET, bucketName, newSubResourceSerial(SubResourceTagging), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketTagging(bucketName string) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doActionWithBucket("DeleteBucketTagging", HTTP_DELETE, bucketName, newSubResourceSerial(SubResourceTagging), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketNotification(input *SetBucketNotificationInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetBucketNotificationInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucket("SetBucketNotification", HTTP_PUT, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketNotification(bucketName string) (output *GetBucketNotificationOutput, err error) { + if obsClient.conf.signature != SignatureObs { + return obsClient.getBucketNotificationS3(bucketName) + } + output = &GetBucketNotificationOutput{} + err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) getBucketNotificationS3(bucketName string) (output *GetBucketNotificationOutput, err error) { + outputS3 := &getBucketNotificationOutputS3{} + err = obsClient.doActionWithBucket("GetBucketNotification", HTTP_GET, bucketName, newSubResourceSerial(SubResourceNotification), outputS3) + if err != nil { + return nil, err + } + + output = &GetBucketNotificationOutput{} + output.BaseModel = outputS3.BaseModel + topicConfigurations := make([]TopicConfiguration, 0, len(outputS3.TopicConfigurations)) + for _, topicConfigurationS3 := range outputS3.TopicConfigurations { + topicConfiguration := TopicConfiguration{} + topicConfiguration.ID = topicConfigurationS3.ID + topicConfiguration.Topic = topicConfigurationS3.Topic + topicConfiguration.FilterRules = topicConfigurationS3.FilterRules + + events := make([]EventType, 0, len(topicConfigurationS3.Events)) + for _, event := range topicConfigurationS3.Events { + events = append(events, ParseStringToEventType(event)) + } + topicConfiguration.Events = events + topicConfigurations = append(topicConfigurations, topicConfiguration) + } + output.TopicConfigurations = topicConfigurations + return +} + +func (obsClient ObsClient) DeleteObject(input *DeleteObjectInput) (output *DeleteObjectOutput, err error) { + if input == nil { + return nil, errors.New("DeleteObjectInput is nil") + } + output = &DeleteObjectOutput{} + err = obsClient.doActionWithBucketAndKey("DeleteObject", HTTP_DELETE, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseDeleteObjectOutput(output) + } + return +} + +func (obsClient ObsClient) DeleteObjects(input *DeleteObjectsInput) (output *DeleteObjectsOutput, err error) { + if input == nil { + return nil, errors.New("DeleteObjectsInput is nil") + } + output = &DeleteObjectsOutput{} + err = obsClient.doActionWithBucket("DeleteObjects", HTTP_POST, input.Bucket, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetObjectAcl(input *SetObjectAclInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("SetObjectAclInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("SetObjectAcl", HTTP_PUT, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetObjectAcl(input *GetObjectAclInput) (output *GetObjectAclOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectAclInput is nil") + } + output = &GetObjectAclOutput{} + err = obsClient.doActionWithBucketAndKey("GetObjectAcl", HTTP_GET, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionId[0] + } + } + return +} + +func (obsClient ObsClient) RestoreObject(input *RestoreObjectInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("RestoreObjectInput is nil") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("RestoreObject", HTTP_POST, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetObjectMetadata(input *GetObjectMetadataInput) (output *GetObjectMetadataOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectMetadataInput is nil") + } + output = &GetObjectMetadataOutput{} + err = obsClient.doActionWithBucketAndKey("GetObjectMetadata", HTTP_HEAD, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseGetObjectMetadataOutput(output) + } + return +} + +func (obsClient ObsClient) GetObject(input *GetObjectInput) (output *GetObjectOutput, err error) { + if input == nil { + return nil, errors.New("GetObjectInput is nil") + } + output = &GetObjectOutput{} + err = obsClient.doActionWithBucketAndKey("GetObject", HTTP_GET, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseGetObjectOutput(output) + } + return +} + +func (obsClient ObsClient) PutObject(input *PutObjectInput) (output *PutObjectOutput, err error) { + if input == nil { + return nil, errors.New("PutObjectInput is nil") + } + + if input.ContentType == "" && input.Key != "" { + if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok { + input.ContentType = contentType + } + } + + output = &PutObjectOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if input.ContentLength > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.ContentLength} + } + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("PutObject", HTTP_PUT, input.Bucket, input.Key, input, output) + } + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +func (obsClient ObsClient) PutFile(input *PutFileInput) (output *PutObjectOutput, err error) { + if input == nil { + return nil, errors.New("PutFileInput is nil") + } + + var body io.Reader + sourceFile := strings.TrimSpace(input.SourceFile) + if sourceFile != "" { + fd, err := os.Open(sourceFile) + if err != nil { + return nil, err + } + defer fd.Close() + + stat, err := fd.Stat() + if err != nil { + return nil, err + } + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + if input.ContentLength > 0 { + if input.ContentLength > stat.Size() { + input.ContentLength = stat.Size() + } + fileReaderWrapper.totalCount = input.ContentLength + } else { + fileReaderWrapper.totalCount = stat.Size() + } + body = fileReaderWrapper + } + + _input := &PutObjectInput{} + _input.PutObjectBasicInput = input.PutObjectBasicInput + _input.Body = body + + if _input.ContentType == "" && _input.Key != "" { + if contentType, ok := mime_types[_input.Key[strings.LastIndex(_input.Key, ".")+1:]]; ok { + _input.ContentType = contentType + } else if contentType, ok := mime_types[sourceFile[strings.LastIndex(sourceFile, ".")+1:]]; ok { + _input.ContentType = contentType + } + } + + output = &PutObjectOutput{} + err = obsClient.doActionWithBucketAndKey("PutFile", HTTP_PUT, _input.Bucket, _input.Key, _input, output) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +func (obsClient ObsClient) CopyObject(input *CopyObjectInput) (output *CopyObjectOutput, err error) { + if input == nil { + return nil, errors.New("CopyObjectInput is nil") + } + + if strings.TrimSpace(input.CopySourceBucket) == "" { + return nil, errors.New("Source bucket is empty") + } + if strings.TrimSpace(input.CopySourceKey) == "" { + return nil, errors.New("Source key is empty") + } + + output = &CopyObjectOutput{} + err = obsClient.doActionWithBucketAndKey("CopyObject", HTTP_PUT, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseCopyObjectOutput(output) + } + return +} + +func (obsClient ObsClient) AbortMultipartUpload(input *AbortMultipartUploadInput) (output *BaseModel, err error) { + if input == nil { + return nil, errors.New("AbortMultipartUploadInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + output = &BaseModel{} + err = obsClient.doActionWithBucketAndKey("AbortMultipartUpload", HTTP_DELETE, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploadInput) (output *InitiateMultipartUploadOutput, err error) { + if input == nil { + return nil, errors.New("InitiateMultipartUploadInput is nil") + } + + if input.ContentType == "" && input.Key != "" { + if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok { + input.ContentType = contentType + } + } + + output = &InitiateMultipartUploadOutput{} + err = obsClient.doActionWithBucketAndKey("InitiateMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseInitiateMultipartUploadOutput(output) + } + return +} + +func (obsClient ObsClient) UploadPart(input *UploadPartInput) (output *UploadPartOutput, err error) { + if input == nil { + return nil, errors.New("UploadPartInput is nil") + } + + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + + output = &UploadPartOutput{} + var repeatable bool + if input.Body != nil { + _, repeatable = input.Body.(*strings.Reader) + if input.PartSize > 0 { + input.Body = &readerWrapper{reader: input.Body, totalCount: input.PartSize} + } + } else if sourceFile := strings.TrimSpace(input.SourceFile); sourceFile != "" { + fd, err := os.Open(sourceFile) + if err != nil { + return nil, err + } + defer fd.Close() + + stat, err := fd.Stat() + if err != nil { + return nil, err + } + fileSize := stat.Size() + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + + if input.Offset < 0 || input.Offset > fileSize { + input.Offset = 0 + } + + if input.PartSize <= 0 || input.PartSize > (fileSize-input.Offset) { + input.PartSize = fileSize - input.Offset + } + fileReaderWrapper.totalCount = input.PartSize + fd.Seek(input.Offset, 0) + input.Body = fileReaderWrapper + repeatable = true + } + if repeatable { + err = obsClient.doActionWithBucketAndKey("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output) + } else { + err = obsClient.doActionWithBucketAndKeyUnRepeatable("UploadPart", HTTP_PUT, input.Bucket, input.Key, input, output) + } + if err != nil { + output = nil + } else { + ParseUploadPartOutput(output) + output.PartNumber = input.PartNumber + } + return +} + +func (obsClient ObsClient) CompleteMultipartUpload(input *CompleteMultipartUploadInput) (output *CompleteMultipartUploadOutput, err error) { + if input == nil { + return nil, errors.New("CompleteMultipartUploadInput is nil") + } + + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + + var parts partSlice = input.Parts + sort.Sort(parts) + + output = &CompleteMultipartUploadOutput{} + err = obsClient.doActionWithBucketAndKey("CompleteMultipartUpload", HTTP_POST, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseCompleteMultipartUploadOutput(output) + } + return +} + +func (obsClient ObsClient) ListParts(input *ListPartsInput) (output *ListPartsOutput, err error) { + if input == nil { + return nil, errors.New("ListPartsInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + output = &ListPartsOutput{} + err = obsClient.doActionWithBucketAndKey("ListParts", HTTP_GET, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) CopyPart(input *CopyPartInput) (output *CopyPartOutput, err error) { + if input == nil { + return nil, errors.New("CopyPartInput is nil") + } + if input.UploadId == "" { + return nil, errors.New("UploadId is empty") + } + if strings.TrimSpace(input.CopySourceBucket) == "" { + return nil, errors.New("Source bucket is empty") + } + if strings.TrimSpace(input.CopySourceKey) == "" { + return nil, errors.New("Source key is empty") + } + + output = &CopyPartOutput{} + err = obsClient.doActionWithBucketAndKey("CopyPart", HTTP_PUT, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseCopyPartOutput(output) + output.PartNumber = input.PartNumber + } + return +} diff --git a/pkg/util/huawei/obs/conf.go b/pkg/util/huawei/obs/conf.go new file mode 100644 index 0000000000..a4b695c68e --- /dev/null +++ b/pkg/util/huawei/obs/conf.go @@ -0,0 +1,375 @@ +package obs + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +type securityProvider struct { + ak string + sk string + securityToken string +} + +type urlHolder struct { + scheme string + host string + port int +} + +type config struct { + securityProvider *securityProvider + urlHolder *urlHolder + endpoint string + signature SignatureType + pathStyle bool + region string + connectTimeout int + socketTimeout int + headerTimeout int + idleConnTimeout int + finalTimeout int + maxRetryCount int + proxyUrl string + maxConnsPerHost int + sslVerify bool + pemCerts []byte + transport *http.Transport + ctx context.Context +} + +func (conf config) String() string { + return fmt.Sprintf("[endpoint:%s, signature:%s, pathStyle:%v, region:%s"+ + "\nconnectTimeout:%d, socketTimeout:%dheaderTimeout:%d, idleConnTimeout:%d"+ + "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, proxyUrl:%s]", + conf.endpoint, conf.signature, conf.pathStyle, conf.region, + conf.connectTimeout, conf.socketTimeout, conf.headerTimeout, conf.idleConnTimeout, + conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.proxyUrl, + ) +} + +type configurer func(conf *config) + +func WithSslVerify(sslVerify bool) configurer { + return WithSslVerifyAndPemCerts(sslVerify, nil) +} + +func WithSslVerifyAndPemCerts(sslVerify bool, pemCerts []byte) configurer { + return func(conf *config) { + conf.sslVerify = sslVerify + conf.pemCerts = pemCerts + } +} + +func WithHeaderTimeout(headerTimeout int) configurer { + return func(conf *config) { + conf.headerTimeout = headerTimeout + } +} + +func WithProxyUrl(proxyUrl string) configurer { + return func(conf *config) { + conf.proxyUrl = proxyUrl + } +} + +func WithMaxConnections(maxConnsPerHost int) configurer { + return func(conf *config) { + conf.maxConnsPerHost = maxConnsPerHost + } +} + +func WithPathStyle(pathStyle bool) configurer { + return func(conf *config) { + conf.pathStyle = pathStyle + } +} + +func WithSignature(signature SignatureType) configurer { + return func(conf *config) { + conf.signature = signature + } +} + +func WithRegion(region string) configurer { + return func(conf *config) { + conf.region = region + } +} + +func WithConnectTimeout(connectTimeout int) configurer { + return func(conf *config) { + conf.connectTimeout = connectTimeout + } +} + +func WithSocketTimeout(socketTimeout int) configurer { + return func(conf *config) { + conf.socketTimeout = socketTimeout + } +} + +func WithIdleConnTimeout(idleConnTimeout int) configurer { + return func(conf *config) { + conf.idleConnTimeout = idleConnTimeout + } +} + +func WithMaxRetryCount(maxRetryCount int) configurer { + return func(conf *config) { + conf.maxRetryCount = maxRetryCount + } +} + +func WithSecurityToken(securityToken string) configurer { + return func(conf *config) { + conf.securityProvider.securityToken = securityToken + } +} + +func WithHttpTransport(transport *http.Transport) configurer { + return func(conf *config) { + conf.transport = transport + } +} + +func WithRequestContext(ctx context.Context) configurer { + return func(conf *config) { + conf.ctx = ctx + } +} + +func (conf *config) initConfigWithDefault() error { + conf.securityProvider.ak = strings.TrimSpace(conf.securityProvider.ak) + conf.securityProvider.sk = strings.TrimSpace(conf.securityProvider.sk) + conf.securityProvider.securityToken = strings.TrimSpace(conf.securityProvider.securityToken) + conf.endpoint = strings.TrimSpace(conf.endpoint) + if conf.endpoint == "" { + return errors.New("endpoint is not set") + } + + if index := strings.Index(conf.endpoint, "?"); index > 0 { + conf.endpoint = conf.endpoint[:index] + } + + for strings.LastIndex(conf.endpoint, "/") == len(conf.endpoint)-1 { + conf.endpoint = conf.endpoint[:len(conf.endpoint)-1] + } + + if conf.signature == "" { + conf.signature = DEFAULT_SIGNATURE + } + + urlHolder := &urlHolder{} + var address string + if strings.HasPrefix(conf.endpoint, "https://") { + urlHolder.scheme = "https" + address = conf.endpoint[len("https://"):] + } else if strings.HasPrefix(conf.endpoint, "http://") { + urlHolder.scheme = "http" + address = conf.endpoint[len("http://"):] + } else { + urlHolder.scheme = "http" + address = conf.endpoint + } + + addr := strings.Split(address, ":") + if len(addr) == 2 { + if port, err := strconv.Atoi(addr[1]); err == nil { + urlHolder.port = port + } + } + urlHolder.host = addr[0] + if urlHolder.port == 0 { + if urlHolder.scheme == "https" { + urlHolder.port = 443 + } else { + urlHolder.port = 80 + } + } + + if IsIP(urlHolder.host) { + conf.pathStyle = true + } + + conf.urlHolder = urlHolder + + conf.region = strings.TrimSpace(conf.region) + if conf.region == "" { + conf.region = DEFAULT_REGION + } + + if conf.connectTimeout <= 0 { + conf.connectTimeout = DEFAULT_CONNECT_TIMEOUT + } + + if conf.socketTimeout <= 0 { + conf.socketTimeout = DEFAULT_SOCKET_TIMEOUT + } + + conf.finalTimeout = conf.socketTimeout * 10 + + if conf.headerTimeout <= 0 { + conf.headerTimeout = DEFAULT_HEADER_TIMEOUT + } + + if conf.idleConnTimeout < 0 { + conf.idleConnTimeout = DEFAULT_IDLE_CONN_TIMEOUT + } + + if conf.maxRetryCount < 0 { + conf.maxRetryCount = DEFAULT_MAX_RETRY_COUNT + } + + if conf.maxConnsPerHost <= 0 { + conf.maxConnsPerHost = DEFAULT_MAX_CONN_PER_HOST + } + + conf.proxyUrl = strings.TrimSpace(conf.proxyUrl) + return nil +} + +func (conf *config) getTransport() error { + if conf.transport == nil { + conf.transport = &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(network, addr, time.Second*time.Duration(conf.connectTimeout)) + if err != nil { + return nil, err + } + return getConnDelegate(conn, conf.socketTimeout, conf.finalTimeout), nil + }, + MaxIdleConns: conf.maxConnsPerHost, + MaxIdleConnsPerHost: conf.maxConnsPerHost, + ResponseHeaderTimeout: time.Second * time.Duration(conf.headerTimeout), + IdleConnTimeout: time.Second * time.Duration(conf.idleConnTimeout), + } + + if conf.proxyUrl != "" { + proxyUrl, err := url.Parse(conf.proxyUrl) + if err != nil { + return err + } + conf.transport.Proxy = http.ProxyURL(proxyUrl) + } + + tlsConfig := &tls.Config{InsecureSkipVerify: !conf.sslVerify} + if conf.sslVerify && conf.pemCerts != nil { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(conf.pemCerts) + tlsConfig.RootCAs = pool + } + + conf.transport.TLSClientConfig = tlsConfig + } + + return nil +} + +func checkRedirectFunc(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse +} + +func DummyQueryEscape(s string) string { + return s +} + +func (conf *config) formatUrls(bucketName, objectKey string, params map[string]string, escape bool) (requestUrl string, canonicalizedUrl string) { + + urlHolder := conf.urlHolder + if bucketName == "" { + requestUrl = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) + canonicalizedUrl = "/" + } else { + if conf.pathStyle { + requestUrl = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName) + canonicalizedUrl = "/" + bucketName + } else { + requestUrl = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port) + if conf.signature == "v2" { + canonicalizedUrl = "/" + bucketName + "/" + } else { + canonicalizedUrl = "/" + } + } + } + var escapeFunc func(s string) string + if escape { + escapeFunc = url.QueryEscape + } else { + escapeFunc = DummyQueryEscape + } + + if objectKey != "" { + encodeObjectKey := escapeFunc(objectKey) + requestUrl += "/" + encodeObjectKey + if !strings.HasSuffix(canonicalizedUrl, "/") { + canonicalizedUrl += "/" + } + canonicalizedUrl += encodeObjectKey + } + + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, strings.TrimSpace(key)) + } + sort.Strings(keys) + i := 0 + + for index, key := range keys { + if index == 0 { + requestUrl += "?" + } else { + requestUrl += "&" + } + _key := url.QueryEscape(key) + requestUrl += _key + + _value := params[key] + if conf.signature == "v4" { + requestUrl += "=" + url.QueryEscape(_value) + } else { + if _value != "" { + requestUrl += "=" + url.QueryEscape(_value) + _value = "=" + _value + } else { + _value = "" + } + lowerKey := strings.ToLower(key) + _, ok := allowed_resource_parameter_names[lowerKey] + prefixHeader := HEADER_PREFIX + isObs := conf.signature == SignatureObs + if isObs { + prefixHeader = HEADER_PREFIX_OBS + } + ok = ok || strings.HasPrefix(lowerKey, prefixHeader) + if ok { + if i == 0 { + canonicalizedUrl += "?" + } else { + canonicalizedUrl += "&" + } + canonicalizedUrl += getQueryUrl(_key, _value) + i++ + } + } + } + return +} + +func getQueryUrl(key, value string) string { + queryUrl := "" + queryUrl += key + queryUrl += value + return queryUrl +} diff --git a/pkg/util/huawei/obs/const.go b/pkg/util/huawei/obs/const.go new file mode 100644 index 0000000000..e3b3646058 --- /dev/null +++ b/pkg/util/huawei/obs/const.go @@ -0,0 +1,493 @@ +package obs + +const ( + obs_sdk_version = "3.0.0" + USER_AGENT = "obs-sdk-go/" + obs_sdk_version + HEADER_PREFIX = "x-amz-" + HEADER_PREFIX_META = "x-amz-meta-" + HEADER_PREFIX_OBS = "x-obs-" + HEADER_PREFIX_META_OBS = "x-obs-meta-" + HEADER_DATE_AMZ = "x-amz-date" + HEADER_DATE_OBS = "x-obs-date" + HEADER_STS_TOKEN_AMZ = "x-amz-security-token" + HEADER_ACCESSS_KEY_AMZ = "AWSAccessKeyId" + PREFIX_META = "meta-" + + HEADER_CONTENT_SHA256_AMZ = "x-amz-content-sha256" + HEADER_ACL_AMZ = "x-amz-acl" + HEADER_ACL_OBS = "x-obs-acl" + HEADER_ACL = "acl" + HEADER_LOCATION_AMZ = "location" + HEADER_BUCKET_LOCATION_OBS = "bucket-location" + HEADER_COPY_SOURCE = "copy-source" + HEADER_COPY_SOURCE_RANGE = "copy-source-range" + HEADER_RANGE = "Range" + HEADER_STORAGE_CLASS = "x-default-storage-class" + HEADER_STORAGE_CLASS_OBS = "x-obs-storage-class" + HEADER_VERSION_OBS = "version" + HEADER_GRANT_READ_OBS = "grant-read" + HEADER_GRANT_WRITE_OBS = "grant-write" + HEADER_GRANT_READ_ACP_OBS = "grant-read-acp" + HEADER_GRANT_WRITE_ACP_OBS = "grant-write-acp" + HEADER_GRANT_FULL_CONTROL_OBS = "grant-full-control" + HEADER_GRANT_READ_DELIVERED_OBS = "grant-read-delivered" + HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS = "grant-full-control-delivered" + HEADER_REQUEST_ID = "request-id" + HEADER_BUCKET_REGION = "bucket-region" + HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN = "access-control-allow-origin" + HEADER_ACCESS_CONRTOL_ALLOW_HEADERS = "access-control-allow-headers" + HEADER_ACCESS_CONRTOL_MAX_AGE = "access-control-max-age" + HEADER_ACCESS_CONRTOL_ALLOW_METHODS = "access-control-allow-methods" + HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS = "access-control-expose-headers" + HEADER_EPID_HEADERS = "epid" + HEADER_VERSION_ID = "version-id" + HEADER_COPY_SOURCE_VERSION_ID = "copy-source-version-id" + HEADER_DELETE_MARKER = "delete-marker" + HEADER_WEBSITE_REDIRECT_LOCATION = "website-redirect-location" + HEADER_METADATA_DIRECTIVE = "metadata-directive" + HEADER_EXPIRATION = "expiration" + HEADER_EXPIRES_OBS = "x-obs-expires" + HEADER_RESTORE = "restore" + HEADER_OBJECT_TYPE = "object-type" + HEADER_NEXT_APPEND_POSITION = "next-append-position" + HEADER_STORAGE_CLASS2 = "storage-class" + HEADER_CONTENT_LENGTH = "content-length" + HEADER_CONTENT_TYPE = "content-type" + HEADER_CONTENT_LANGUAGE = "content-language" + HEADER_EXPIRES = "expires" + HEADER_CACHE_CONTROL = "cache-control" + HEADER_CONTENT_DISPOSITION = "content-disposition" + HEADER_CONTENT_ENCODING = "content-encoding" + + HEADER_ETAG = "etag" + HEADER_LASTMODIFIED = "last-modified" + + HEADER_COPY_SOURCE_IF_MATCH = "copy-source-if-match" + HEADER_COPY_SOURCE_IF_NONE_MATCH = "copy-source-if-none-match" + HEADER_COPY_SOURCE_IF_MODIFIED_SINCE = "copy-source-if-modified-since" + HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE = "copy-source-if-unmodified-since" + + HEADER_IF_MATCH = "If-Match" + HEADER_IF_NONE_MATCH = "If-None-Match" + HEADER_IF_MODIFIED_SINCE = "If-Modified-Since" + HEADER_IF_UNMODIFIED_SINCE = "If-Unmodified-Since" + + HEADER_SSEC_ENCRYPTION = "server-side-encryption-customer-algorithm" + HEADER_SSEC_KEY = "server-side-encryption-customer-key" + HEADER_SSEC_KEY_MD5 = "server-side-encryption-customer-key-MD5" + + HEADER_SSEKMS_ENCRYPTION = "server-side-encryption" + HEADER_SSEKMS_KEY = "server-side-encryption-aws-kms-key-id" + HEADER_SSEKMS_ENCRYPT_KEY_OBS = "server-side-encryption-kms-key-id" + + HEADER_SSEC_COPY_SOURCE_ENCRYPTION = "copy-source-server-side-encryption-customer-algorithm" + HEADER_SSEC_COPY_SOURCE_KEY = "copy-source-server-side-encryption-customer-key" + HEADER_SSEC_COPY_SOURCE_KEY_MD5 = "copy-source-server-side-encryption-customer-key-MD5" + + HEADER_SSEKMS_KEY_AMZ = "x-amz-server-side-encryption-aws-kms-key-id" + + HEADER_SSEKMS_KEY_OBS = "x-obs-server-side-encryption-kms-key-id" + + HEADER_SUCCESS_ACTION_REDIRECT = "success_action_redirect" + + HEADER_DATE_CAMEL = "Date" + HEADER_HOST_CAMEL = "Host" + HEADER_HOST = "host" + HEADER_AUTH_CAMEL = "Authorization" + HEADER_MD5_CAMEL = "Content-MD5" + HEADER_LOCATION_CAMEL = "Location" + HEADER_CONTENT_LENGTH_CAMEL = "Content-Length" + HEADER_CONTENT_TYPE_CAML = "Content-Type" + HEADER_USER_AGENT_CAMEL = "User-Agent" + HEADER_ORIGIN_CAMEL = "Origin" + HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL = "Access-Control-Request-Headers" + + PARAM_VERSION_ID = "versionId" + PARAM_RESPONSE_CONTENT_TYPE = "response-content-type" + PARAM_RESPONSE_CONTENT_LANGUAGE = "response-content-language" + PARAM_RESPONSE_EXPIRES = "response-expires" + PARAM_RESPONSE_CACHE_CONTROL = "response-cache-control" + PARAM_RESPONSE_CONTENT_DISPOSITION = "response-content-disposition" + PARAM_RESPONSE_CONTENT_ENCODING = "response-content-encoding" + PARAM_IMAGE_PROCESS = "x-image-process" + + PARAM_ALGORITHM_AMZ_CAMEL = "X-Amz-Algorithm" + PARAM_CREDENTIAL_AMZ_CAMEL = "X-Amz-Credential" + PARAM_DATE_AMZ_CAMEL = "X-Amz-Date" + PARAM_DATE_OBS_CAMEL = "X-Obs-Date" + PARAM_EXPIRES_AMZ_CAMEL = "X-Amz-Expires" + PARAM_SIGNEDHEADERS_AMZ_CAMEL = "X-Amz-SignedHeaders" + PARAM_SIGNATURE_AMZ_CAMEL = "X-Amz-Signature" + + DEFAULT_SIGNATURE = SignatureV2 + DEFAULT_REGION = "region" + DEFAULT_CONNECT_TIMEOUT = 60 + DEFAULT_SOCKET_TIMEOUT = 60 + DEFAULT_HEADER_TIMEOUT = 60 + DEFAULT_IDLE_CONN_TIMEOUT = 30 + DEFAULT_MAX_RETRY_COUNT = 3 + DEFAULT_MAX_CONN_PER_HOST = 1000 + EMPTY_CONTENT_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD" + LONG_DATE_FORMAT = "20060102T150405Z" + SHORT_DATE_FORMAT = "20060102" + ISO8601_DATE_FORMAT = "2006-01-02T15:04:05Z" + ISO8601_MIDNIGHT_DATE_FORMAT = "2006-01-02T00:00:00Z" + RFC1123_FORMAT = "Mon, 02 Jan 2006 15:04:05 GMT" + + V4_SERVICE_NAME = "s3" + V4_SERVICE_SUFFIX = "aws4_request" + + V2_HASH_PREFIX = "AWS" + OBS_HASH_PREFIX = "OBS" + + V4_HASH_PREFIX = "AWS4-HMAC-SHA256" + V4_HASH_PRE = "AWS4" + + DEFAULT_SSE_KMS_ENCRYPTION = "aws:kms" + DEFAULT_SSE_KMS_ENCRYPTION_OBS = "kms" + + DEFAULT_SSE_C_ENCRYPTION = "AES256" + + HTTP_GET = "GET" + HTTP_POST = "POST" + HTTP_PUT = "PUT" + HTTP_DELETE = "DELETE" + HTTP_HEAD = "HEAD" + HTTP_OPTIONS = "OPTIONS" +) + +type SignatureType string + +const ( + SignatureV2 SignatureType = "v2" + SignatureV4 SignatureType = "v4" + SignatureObs SignatureType = "OBS" +) + +var ( + interested_headers = []string{"content-md5", "content-type", "date"} + + allowed_response_http_header_metadata_names = map[string]bool{ + "content-type": true, + "content-md5": true, + "content-length": true, + "content-language": true, + "expires": true, + "origin": true, + "cache-control": true, + "content-disposition": true, + "content-encoding": true, + "x-default-storage-class": true, + "location": true, + "date": true, + "etag": true, + "host": true, + "last-modified": true, + "content-range": true, + "x-reserved": true, + "access-control-allow-origin": true, + "access-control-allow-headers": true, + "access-control-max-age": true, + "access-control-allow-methods": true, + "access-control-expose-headers": true, + "connection": true, + } + + allowed_request_http_header_metadata_names = map[string]bool{ + "content-type": true, + "content-md5": true, + "content-length": true, + "content-language": true, + "expires": true, + "origin": true, + "cache-control": true, + "content-disposition": true, + "content-encoding": true, + "access-control-request-method": true, + "access-control-request-headers": true, + "x-default-storage-class": true, + "location": true, + "date": true, + "etag": true, + "range": true, + "host": true, + "if-modified-since": true, + "if-unmodified-since": true, + "if-match": true, + "if-none-match": true, + "last-modified": true, + "content-range": true, + } + + allowed_resource_parameter_names = map[string]bool{ + "acl": true, + "backtosource": true, + "policy": true, + "torrent": true, + "logging": true, + "location": true, + "storageinfo": true, + "quota": true, + "storageclass": true, + "storagepolicy": true, + "requestpayment": true, + "versions": true, + "versioning": true, + "versionid": true, + "uploads": true, + "uploadid": true, + "partnumber": true, + "website": true, + "notification": true, + "lifecycle": true, + "deletebucket": true, + "delete": true, + "cors": true, + "restore": true, + "tagging": true, + "append": true, + "position": true, + "replication": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, + "x-image-process": true, + "x-oss-process": true, + } + + mime_types = map[string]string{ + "7z": "application/x-7z-compressed", + "aac": "audio/x-aac", + "ai": "application/postscript", + "aif": "audio/x-aiff", + "asc": "text/plain", + "asf": "video/x-ms-asf", + "atom": "application/atom+xml", + "avi": "video/x-msvideo", + "bmp": "image/bmp", + "bz2": "application/x-bzip2", + "cer": "application/pkix-cert", + "crl": "application/pkix-crl", + "crt": "application/x-x509-ca-cert", + "css": "text/css", + "csv": "text/csv", + "cu": "application/cu-seeme", + "deb": "application/x-debian-package", + "doc": "application/msword", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dvi": "application/x-dvi", + "eot": "application/vnd.ms-fontobject", + "eps": "application/postscript", + "epub": "application/epub+zip", + "etx": "text/x-setext", + "flac": "audio/flac", + "flv": "video/x-flv", + "gif": "image/gif", + "gz": "application/gzip", + "htm": "text/html", + "html": "text/html", + "ico": "image/x-icon", + "ics": "text/calendar", + "ini": "text/plain", + "iso": "application/x-iso9660-image", + "jar": "application/java-archive", + "jpe": "image/jpeg", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "js": "text/javascript", + "json": "application/json", + "latex": "application/x-latex", + "log": "text/plain", + "m4a": "audio/mp4", + "m4v": "video/mp4", + "mid": "audio/midi", + "midi": "audio/midi", + "mov": "video/quicktime", + "mp3": "audio/mpeg", + "mp4": "video/mp4", + "mp4a": "audio/mp4", + "mp4v": "video/mp4", + "mpe": "video/mpeg", + "mpeg": "video/mpeg", + "mpg": "video/mpeg", + "mpg4": "video/mp4", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "ogv": "video/ogg", + "ogx": "application/ogg", + "pbm": "image/x-portable-bitmap", + "pdf": "application/pdf", + "pgm": "image/x-portable-graymap", + "png": "image/png", + "pnm": "image/x-portable-anymap", + "ppm": "image/x-portable-pixmap", + "ppt": "application/vnd.ms-powerpoint", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "ps": "application/postscript", + "qt": "video/quicktime", + "rar": "application/x-rar-compressed", + "ras": "image/x-cmu-raster", + "rss": "application/rss+xml", + "rtf": "application/rtf", + "sgm": "text/sgml", + "sgml": "text/sgml", + "svg": "image/svg+xml", + "swf": "application/x-shockwave-flash", + "tar": "application/x-tar", + "tif": "image/tiff", + "tiff": "image/tiff", + "torrent": "application/x-bittorrent", + "ttf": "application/x-font-ttf", + "txt": "text/plain", + "wav": "audio/x-wav", + "webm": "video/webm", + "wma": "audio/x-ms-wma", + "wmv": "video/x-ms-wmv", + "woff": "application/x-font-woff", + "wsdl": "application/wsdl+xml", + "xbm": "image/x-xbitmap", + "xls": "application/vnd.ms-excel", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xml": "application/xml", + "xpm": "image/x-xpixmap", + "xwd": "image/x-xwindowdump", + "yaml": "text/yaml", + "yml": "text/yaml", + "zip": "application/zip", + } +) + +type HttpMethodType string + +const ( + HttpMethodGet HttpMethodType = HTTP_GET + HttpMethodPut HttpMethodType = HTTP_PUT + HttpMethodPost HttpMethodType = HTTP_POST + HttpMethodDelete HttpMethodType = HTTP_DELETE + HttpMethodHead HttpMethodType = HTTP_HEAD + HttpMethodOptions HttpMethodType = HTTP_OPTIONS +) + +type SubResourceType string + +const ( + SubResourceStoragePolicy SubResourceType = "storagePolicy" + SubResourceStorageClass SubResourceType = "storageClass" + SubResourceQuota SubResourceType = "quota" + SubResourceStorageInfo SubResourceType = "storageinfo" + SubResourceLocation SubResourceType = "location" + SubResourceAcl SubResourceType = "acl" + SubResourcePolicy SubResourceType = "policy" + SubResourceCors SubResourceType = "cors" + SubResourceVersioning SubResourceType = "versioning" + SubResourceWebsite SubResourceType = "website" + SubResourceLogging SubResourceType = "logging" + SubResourceLifecycle SubResourceType = "lifecycle" + SubResourceNotification SubResourceType = "notification" + SubResourceTagging SubResourceType = "tagging" + SubResourceDelete SubResourceType = "delete" + SubResourceVersions SubResourceType = "versions" + SubResourceUploads SubResourceType = "uploads" + SubResourceRestore SubResourceType = "restore" +) + +type AclType string + +const ( + AclPrivate AclType = "private" + AclPublicRead AclType = "public-read" + AclPublicReadWrite AclType = "public-read-write" + AclAuthenticatedRead AclType = "authenticated-read" + AclBucketOwnerRead AclType = "bucket-owner-read" + AclBucketOwnerFullControl AclType = "bucket-owner-full-control" + AclLogDeliveryWrite AclType = "log-delivery-write" + AclPublicReadDelivery AclType = "public-read-delivered" + AclPublicReadWriteDelivery AclType = "public-read-write-delivered" +) + +type StorageClassType string + +const ( + StorageClassStandard StorageClassType = "STANDARD" + StorageClassWarm StorageClassType = "STANDARD_IA" + StorageClassCold StorageClassType = "GLACIER" +) + +type PermissionType string + +const ( + PermissionRead PermissionType = "READ" + PermissionWrite PermissionType = "WRITE" + PermissionReadAcp PermissionType = "READ_ACP" + PermissionWriteAcp PermissionType = "WRITE_ACP" + PermissionFullControl PermissionType = "FULL_CONTROL" +) + +type GranteeType string + +const ( + GranteeGroup GranteeType = "Group" + GranteeUser GranteeType = "CanonicalUser" +) + +type GroupUriType string + +const ( + GroupAllUsers GroupUriType = "AllUsers" + GroupAuthenticatedUsers GroupUriType = "AuthenticatedUsers" + GroupLogDelivery GroupUriType = "LogDelivery" +) + +type VersioningStatusType string + +const ( + VersioningStatusEnabled VersioningStatusType = "Enabled" + VersioningStatusSuspended VersioningStatusType = "Suspended" +) + +type ProtocolType string + +const ( + ProtocolHttp ProtocolType = "http" + ProtocolHttps ProtocolType = "https" +) + +type RuleStatusType string + +const ( + RuleStatusEnabled RuleStatusType = "Enabled" + RuleStatusDisabled RuleStatusType = "Disabled" +) + +type RestoreTierType string + +const ( + RestoreTierExpedited RestoreTierType = "Expedited" + RestoreTierStandard RestoreTierType = "Standard" + RestoreTierBulk RestoreTierType = "Bulk" +) + +type MetadataDirectiveType string + +const ( + CopyMetadata MetadataDirectiveType = "COPY" + ReplaceMetadata MetadataDirectiveType = "REPLACE" +) + +type EventType string + +const ( + ObjectCreatedAll EventType = "ObjectCreated:*" + ObjectCreatedPut EventType = "ObjectCreated:Put" + ObjectCreatedPost EventType = "ObjectCreated:Post" + + ObjectCreatedCopy EventType = "ObjectCreated:Copy" + ObjectCreatedCompleteMultipartUpload EventType = "ObjectCreated:CompleteMultipartUpload" + ObjectRemovedAll EventType = "ObjectRemoved:*" + ObjectRemovedDelete EventType = "ObjectRemoved:Delete" + ObjectRemovedDeleteMarkerCreated EventType = "ObjectRemoved:DeleteMarkerCreated" +) diff --git a/pkg/util/huawei/obs/convert.go b/pkg/util/huawei/obs/convert.go new file mode 100644 index 0000000000..abab8f5b86 --- /dev/null +++ b/pkg/util/huawei/obs/convert.go @@ -0,0 +1,699 @@ +package obs + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "reflect" + "strings" + "time" +) + +func cleanHeaderPrefix(header http.Header, isObs bool) map[string][]string { + responseHeaders := make(map[string][]string) + for key, value := range header { + if len(value) > 0 { + key = strings.ToLower(key) + headerPrefix := HEADER_PREFIX + if isObs { + headerPrefix = HEADER_PREFIX_OBS + } + if strings.HasPrefix(key, headerPrefix) { + key = key[len(headerPrefix):] + } + responseHeaders[key] = value + } + } + return responseHeaders +} + +func ParseStringToEventType(value string) (ret EventType) { + switch value { + case "ObjectCreated:*", "s3:ObjectCreated:*": + ret = ObjectCreatedAll + case "ObjectCreated:Put", "s3:ObjectCreated:Put": + ret = ObjectCreatedPut + case "ObjectCreated:Post", "s3:ObjectCreated:Post": + ret = ObjectCreatedPost + case "ObjectCreated:Copy", "s3:ObjectCreated:Copy": + ret = ObjectCreatedCopy + case "ObjectCreated:CompleteMultipartUpload", "s3:ObjectCreated:CompleteMultipartUpload": + ret = ObjectCreatedCompleteMultipartUpload + case "ObjectRemoved:*", "s3:ObjectRemoved:*": + ret = ObjectRemovedAll + case "ObjectRemoved:Delete", "s3:ObjectRemoved:Delete": + ret = ObjectRemovedDelete + case "ObjectRemoved:DeleteMarkerCreated", "s3:ObjectRemoved:DeleteMarkerCreated": + ret = ObjectRemovedDeleteMarkerCreated + default: + ret = "" + } + return +} + +func ParseStringToStorageClassType(value string) (ret StorageClassType) { + switch value { + case "STANDARD": + ret = StorageClassStandard + case "STANDARD_IA", "WARM": + ret = StorageClassWarm + case "GLACIER", "COLD": + ret = StorageClassCold + default: + ret = "" + } + return +} + +func convertGrantToXml(grant Grant, isObs bool) string { + xml := make([]string, 0, 4) + if !isObs { + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + } else { + xml = append(xml, fmt.Sprintf("")) + } + if grant.Grantee.Type == GranteeUser { + if grant.Grantee.ID != "" { + granteeID := XmlTranscoding(grant.Grantee.ID) + xml = append(xml, fmt.Sprintf("%s", granteeID)) + } + if grant.Grantee.DisplayName != "" { + granteeDisplayName := XmlTranscoding(grant.Grantee.DisplayName) + xml = append(xml, fmt.Sprintf("%s", granteeDisplayName)) + } + } else { + if !isObs { + if grant.Grantee.URI == GroupAllUsers || grant.Grantee.URI == GroupAuthenticatedUsers { + xml = append(xml, fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/global/", grant.Grantee.URI)) + } else if grant.Grantee.URI == GroupLogDelivery { + xml = append(xml, fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/s3/", grant.Grantee.URI)) + } else { + xml = append(xml, fmt.Sprintf("%s", grant.Grantee.URI)) + } + } else if grant.Grantee.URI == GroupAllUsers { + xml = append(xml, fmt.Sprintf("Everyone")) + } + } + xml = append(xml, fmt.Sprintf("")) + xml = append(xml, fmt.Sprintf("%s", grant.Permission)) + if isObs { + xml = append(xml, fmt.Sprintf("%t", grant.Delivered)) + } + xml = append(xml, fmt.Sprintf("")) + return strings.Join(xml, "") +} + +func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool, isObs bool) (data string, md5 string) { + grantsLength := len(input.TargetGrants) + xml := make([]string, 0, 8+grantsLength) + + xml = append(xml, "") + if input.Agency != "" { + agency := XmlTranscoding(input.Agency) + xml = append(xml, fmt.Sprintf("%s", agency)) + } + if input.TargetBucket != "" || input.TargetPrefix != "" { + xml = append(xml, "") + xml = append(xml, fmt.Sprintf("%s", input.TargetBucket)) + targetPrefix := XmlTranscoding(input.TargetPrefix) + xml = append(xml, fmt.Sprintf("%s", targetPrefix)) + + if grantsLength > 0 { + xml = append(xml, "") + for _, grant := range input.TargetGrants { + xml = append(xml, convertGrantToXml(grant, isObs)) + } + xml = append(xml, "") + } + + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Grants)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + xml = append(xml, "") + for _, grant := range input.Grants { + xml = append(xml, convertGrantToXml(grant, isObs)) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertConditionToXml(condition Condition) string { + xml := make([]string, 0, 2) + if condition.KeyPrefixEquals != "" { + keyPrefixEquals := XmlTranscoding(condition.KeyPrefixEquals) + xml = append(xml, fmt.Sprintf("%s", keyPrefixEquals)) + } + if condition.HttpErrorCodeReturnedEquals != "" { + xml = append(xml, fmt.Sprintf("%s", condition.HttpErrorCodeReturnedEquals)) + } + if len(xml) > 0 { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + return "" +} + +func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd5 bool) (data string, md5 string) { + routingRuleLength := len(input.RoutingRules) + xml := make([]string, 0, 6+routingRuleLength*10) + xml = append(xml, "") + + if input.RedirectAllRequestsTo.HostName != "" { + xml = append(xml, fmt.Sprintf("%s", input.RedirectAllRequestsTo.HostName)) + if input.RedirectAllRequestsTo.Protocol != "" { + xml = append(xml, fmt.Sprintf("%s", input.RedirectAllRequestsTo.Protocol)) + } + xml = append(xml, "") + } else { + indexDocumentSuffix := XmlTranscoding(input.IndexDocument.Suffix) + xml = append(xml, fmt.Sprintf("%s", indexDocumentSuffix)) + if input.ErrorDocument.Key != "" { + errorDocumentKey := XmlTranscoding(input.ErrorDocument.Key) + xml = append(xml, fmt.Sprintf("%s", errorDocumentKey)) + } + if routingRuleLength > 0 { + xml = append(xml, "") + for _, routingRule := range input.RoutingRules { + xml = append(xml, "") + xml = append(xml, "") + if routingRule.Redirect.Protocol != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.Protocol)) + } + if routingRule.Redirect.HostName != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HostName)) + } + if routingRule.Redirect.ReplaceKeyPrefixWith != "" { + replaceKeyPrefixWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyPrefixWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyPrefixWith)) + } + + if routingRule.Redirect.ReplaceKeyWith != "" { + replaceKeyWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyWith)) + } + if routingRule.Redirect.HttpRedirectCode != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HttpRedirectCode)) + } + xml = append(xml, "") + + if ret := convertConditionToXml(routingRule.Condition); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, "") + } + xml = append(xml, "") + } + } + + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertTransitionsToXml(transitions []Transition, isObs bool) string { + if length := len(transitions); length > 0 { + xml := make([]string, 0, length) + for _, transition := range transitions { + var temp string + if transition.Days > 0 { + temp = fmt.Sprintf("%d", transition.Days) + } else if !transition.Date.IsZero() { + temp = fmt.Sprintf("%s", transition.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT)) + } + if temp != "" { + if !isObs { + storageClass := "STANDARD" + if transition.StorageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if transition.StorageClass == "COLD" { + storageClass = "GLACIER" + } + xml = append(xml, fmt.Sprintf("%s%s", temp, storageClass)) + } else { + xml = append(xml, fmt.Sprintf("%s%s", temp, transition.StorageClass)) + } + } + } + return strings.Join(xml, "") + } + return "" +} + +func convertExpirationToXml(expiration Expiration) string { + if expiration.Days > 0 { + return fmt.Sprintf("%d", expiration.Days) + } else if !expiration.Date.IsZero() { + return fmt.Sprintf("%s", expiration.Date.UTC().Format(ISO8601_MIDNIGHT_DATE_FORMAT)) + } + return "" +} +func convertNoncurrentVersionTransitionsToXml(noncurrentVersionTransitions []NoncurrentVersionTransition, isObs bool) string { + if length := len(noncurrentVersionTransitions); length > 0 { + xml := make([]string, 0, length) + for _, noncurrentVersionTransition := range noncurrentVersionTransitions { + if noncurrentVersionTransition.NoncurrentDays > 0 { + storageClass := string(noncurrentVersionTransition.StorageClass) + if !isObs { + if storageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if storageClass == "COLD" { + storageClass = "GLACIER" + } + } + xml = append(xml, fmt.Sprintf("%d"+ + "%s", + noncurrentVersionTransition.NoncurrentDays, storageClass)) + } + } + return strings.Join(xml, "") + } + return "" +} +func convertNoncurrentVersionExpirationToXml(noncurrentVersionExpiration NoncurrentVersionExpiration) string { + if noncurrentVersionExpiration.NoncurrentDays > 0 { + return fmt.Sprintf("%d", noncurrentVersionExpiration.NoncurrentDays) + } + return "" +} + +func ConvertLifecyleConfigurationToXml(input BucketLifecyleConfiguration, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.LifecycleRules)*9) + xml = append(xml, "") + for _, lifecyleRule := range input.LifecycleRules { + xml = append(xml, "") + if lifecyleRule.ID != "" { + lifecyleRuleID := XmlTranscoding(lifecyleRule.ID) + xml = append(xml, fmt.Sprintf("%s", lifecyleRuleID)) + } + lifecyleRulePrefix := XmlTranscoding(lifecyleRule.Prefix) + xml = append(xml, fmt.Sprintf("%s", lifecyleRulePrefix)) + xml = append(xml, fmt.Sprintf("%s", lifecyleRule.Status)) + if ret := convertTransitionsToXml(lifecyleRule.Transitions, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := convertExpirationToXml(lifecyleRule.Expiration); ret != "" { + xml = append(xml, ret) + } + if ret := convertNoncurrentVersionTransitionsToXml(lifecyleRule.NoncurrentVersionTransitions, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := convertNoncurrentVersionExpirationToXml(lifecyleRule.NoncurrentVersionExpiration); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func converntFilterRulesToXml(filterRules []FilterRule, isObs bool) string { + if length := len(filterRules); length > 0 { + xml := make([]string, 0, length*4) + for _, filterRule := range filterRules { + xml = append(xml, "") + if filterRule.Name != "" { + filterRuleName := XmlTranscoding(filterRule.Name) + xml = append(xml, fmt.Sprintf("%s", filterRuleName)) + } + if filterRule.Value != "" { + filterRuleValue := XmlTranscoding(filterRule.Value) + xml = append(xml, fmt.Sprintf("%s", filterRuleValue)) + } + xml = append(xml, "") + } + if !isObs { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } else { + return fmt.Sprintf("%s", strings.Join(xml, "")) + } + } + return "" +} + +func converntEventsToXml(events []EventType, isObs bool) string { + if length := len(events); length > 0 { + xml := make([]string, 0, length) + if !isObs { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s%s", "s3:", event)) + } + } else { + for _, event := range events { + xml = append(xml, fmt.Sprintf("%s", event)) + } + } + return strings.Join(xml, "") + } + return "" +} + +func converntConfigureToXml(topicConfiguration TopicConfiguration, xmlElem string, isObs bool) string { + xml := make([]string, 0, 6) + xml = append(xml, xmlElem) + if topicConfiguration.ID != "" { + topicConfigurationID := XmlTranscoding(topicConfiguration.ID) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationID)) + } + topicConfigurationTopic := XmlTranscoding(topicConfiguration.Topic) + xml = append(xml, fmt.Sprintf("%s", topicConfigurationTopic)) + + if ret := converntEventsToXml(topicConfiguration.Events, isObs); ret != "" { + xml = append(xml, ret) + } + if ret := converntFilterRulesToXml(topicConfiguration.FilterRules, isObs); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, xmlElem) + return strings.Join(xml, "") +} + +func ConverntObsRestoreToXml(restoreObjectInput RestoreObjectInput) string { + xml := make([]string, 0, 2) + xml = append(xml, fmt.Sprintf("%d", restoreObjectInput.Days)) + if restoreObjectInput.Tier != "Bulk" { + xml = append(xml, fmt.Sprintf("%s", restoreObjectInput.Tier)) + } + xml = append(xml, fmt.Sprintf("")) + data := strings.Join(xml, "") + return data +} + +func ConvertNotificationToXml(input BucketNotification, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.TopicConfigurations)*6) + xml = append(xml, "") + for _, topicConfiguration := range input.TopicConfigurations { + ret := converntConfigureToXml(topicConfiguration, "", isObs) + xml = append(xml, ret) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func ConvertCompleteMultipartUploadInputToXml(input CompleteMultipartUploadInput, returnMd5 bool) (data string, md5 string) { + xml := make([]string, 0, 2+len(input.Parts)*4) + xml = append(xml, "") + for _, part := range input.Parts { + xml = append(xml, "") + xml = append(xml, fmt.Sprintf("%d", part.PartNumber)) + xml = append(xml, fmt.Sprintf("%s", part.ETag)) + xml = append(xml, "") + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func parseSseHeader(responseHeaders map[string][]string) (sseHeader ISseHeader) { + if ret, ok := responseHeaders[HEADER_SSEC_ENCRYPTION]; ok { + sseCHeader := SseCHeader{Encryption: ret[0]} + if ret, ok = responseHeaders[HEADER_SSEC_KEY_MD5]; ok { + sseCHeader.KeyMD5 = ret[0] + } + sseHeader = sseCHeader + } else if ret, ok := responseHeaders[HEADER_SSEKMS_ENCRYPTION]; ok { + sseKmsHeader := SseKmsHeader{Encryption: ret[0]} + if ret, ok = responseHeaders[HEADER_SSEKMS_KEY]; ok { + sseKmsHeader.Key = ret[0] + } else if ret, ok = responseHeaders[HEADER_SSEKMS_ENCRYPT_KEY_OBS]; ok { + sseKmsHeader.Key = ret[0] + } + sseHeader = sseKmsHeader + } + return +} + +func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok { + output.WebsiteRedirectLocation = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EXPIRATION]; ok { + output.Expiration = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_RESTORE]; ok { + output.Restore = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_OBJECT_TYPE]; ok { + output.Restore = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_NEXT_APPEND_POSITION]; ok { + output.Restore = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { + output.ContentType = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { + output.AllowOrigin = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok { + output.AllowHeader = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok { + output.MaxAgeSeconds = StringToInt(ret[0], 0) + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok { + output.AllowMethod = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { + output.ExposeHeader = ret[0] + } + + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_LASTMODIFIED]; ok { + ret, err := time.Parse(time.RFC1123, ret[0]) + if err == nil { + output.LastModified = ret + } + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LENGTH]; ok { + output.ContentLength = StringToInt64(ret[0], 0) + } + + output.Metadata = make(map[string]string) + + for key, value := range output.ResponseHeaders { + if strings.HasPrefix(key, PREFIX_META) { + _key := key[len(PREFIX_META):] + output.ResponseHeaders[_key] = value + output.Metadata[_key] = value[0] + delete(output.ResponseHeaders, key) + } + } + +} + +func ParseCopyObjectOutput(output *CopyObjectOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_COPY_SOURCE_VERSION_ID]; ok { + output.CopySourceVersionId = ret[0] + } +} + +func ParsePutObjectOutput(output *PutObjectOutput) { + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } +} + +func ParseInitiateMultipartUploadOutput(output *InitiateMultipartUploadOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) +} + +func ParseUploadPartOutput(output *UploadPartOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_ETAG]; ok { + output.ETag = ret[0] + } +} + +func ParseCompleteMultipartUploadOutput(output *CompleteMultipartUploadOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) + if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = ret[0] + } +} + +func ParseCopyPartOutput(output *CopyPartOutput) { + output.SseHeader = parseSseHeader(output.ResponseHeaders) +} + +func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS_OBS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_VERSION_OBS]; ok { + output.Version = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = ret[0] + } else if ret, ok := output.ResponseHeaders[HEADER_BUCKET_LOCATION_OBS]; ok { + output.Location = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { + output.AllowOrigin = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok { + output.AllowHeader = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok { + output.MaxAgeSeconds = StringToInt(ret[0], 0) + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok { + output.AllowMethod = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { + output.ExposeHeader = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EPID_HEADERS]; ok { + output.Epid = ret[0] + } +} + +func ParseDeleteObjectOutput(output *DeleteObjectOutput) { + if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionId[0] + } + + if deleteMarker, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok { + output.DeleteMarker = deleteMarker[0] == "true" + } +} + +func ParseGetObjectOutput(output *GetObjectOutput) { + ParseGetObjectMetadataOutput(&output.GetObjectMetadataOutput) + if ret, ok := output.ResponseHeaders[HEADER_DELETE_MARKER]; ok { + output.DeleteMarker = ret[0] == "true" + } + if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok { + output.CacheControl = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok { + output.ContentDisposition = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok { + output.ContentEncoding = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok { + output.ContentLanguage = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok { + output.Expires = ret[0] + } +} + +func ConvertRequestToIoReaderV2(req interface{}) (io.Reader, string, error) { + data, err := TransToXml(req) + if err == nil { + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "Do http request with data: %s", string(data)) + } + return bytes.NewReader(data), Base64Md5(data), nil + } + return nil, "", err +} + +func ConvertRequestToIoReader(req interface{}) (io.Reader, error) { + body, err := TransToXml(req) + if err == nil { + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "Do http request with data: %s", string(body)) + } + return bytes.NewReader(body), nil + } + return nil, err +} + +func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResult bool, isObs bool) (err error) { + readCloser, ok := baseModel.(IReadCloser) + if !ok { + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err == nil && len(body) > 0 { + if xmlResult { + err = ParseXml(body, baseModel) + if err != nil { + doLog(LEVEL_ERROR, "Unmarshal error: %v", err) + } + } else { + s := reflect.TypeOf(baseModel).Elem() + for i := 0; i < s.NumField(); i++ { + if s.Field(i).Tag == "body" { + reflect.ValueOf(baseModel).Elem().FieldByName(s.Field(i).Name).SetString(string(body)) + break + } + } + } + } + } else { + readCloser.setReadCloser(resp.Body) + } + + baseModel.setStatusCode(resp.StatusCode) + responseHeaders := cleanHeaderPrefix(resp.Header, isObs) + baseModel.setResponseHeaders(responseHeaders) + if values, ok := responseHeaders[HEADER_REQUEST_ID]; ok { + baseModel.setRequestId(values[0]) + } + return +} + +func ParseResponseToObsError(resp *http.Response, isObs bool) error { + obsError := ObsError{} + ParseResponseToBaseModel(resp, &obsError, true, isObs) + obsError.Status = resp.Status + return obsError +} diff --git a/pkg/util/huawei/obs/doc.go b/pkg/util/huawei/obs/doc.go new file mode 100644 index 0000000000..a9ecaa2cab --- /dev/null +++ b/pkg/util/huawei/obs/doc.go @@ -0,0 +1 @@ +package obs // import "yunion.io/x/onecloud/pkg/util/huawei/obs" diff --git a/pkg/util/huawei/obs/error.go b/pkg/util/huawei/obs/error.go new file mode 100644 index 0000000000..c79087ec02 --- /dev/null +++ b/pkg/util/huawei/obs/error.go @@ -0,0 +1,21 @@ +package obs + +import ( + "encoding/xml" + "fmt" +) + +type ObsError struct { + BaseModel + Status string + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + Resource string `xml:"Resource"` + HostId string `xml:"HostId"` +} + +func (err ObsError) Error() string { + return fmt.Sprintf("obs: service returned error: Status=%s, Code=%s, Message=%s, RequestId=%s", + err.Status, err.Code, err.Message, err.RequestId) +} diff --git a/pkg/util/huawei/obs/http.go b/pkg/util/huawei/obs/http.go new file mode 100644 index 0000000000..1b7348ec89 --- /dev/null +++ b/pkg/util/huawei/obs/http.go @@ -0,0 +1,429 @@ +package obs + +import ( + "bytes" + "errors" + "io" + "math/rand" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +func prepareHeaders(headers map[string][]string, meta bool, isObs bool) map[string][]string { + _headers := make(map[string][]string, len(headers)) + if headers != nil { + for key, value := range headers { + key = strings.TrimSpace(key) + if key == "" { + continue + } + _key := strings.ToLower(key) + if _, ok := allowed_request_http_header_metadata_names[_key]; !ok && !strings.HasPrefix(key, HEADER_PREFIX) && !strings.HasPrefix(key, HEADER_PREFIX_OBS) { + if !meta { + continue + } + if !isObs { + _key = HEADER_PREFIX_META + _key + } else { + _key = HEADER_PREFIX_META_OBS + _key + } + } else { + _key = key + } + _headers[_key] = value + } + } + return _headers +} + +func (obsClient ObsClient) doActionWithoutBucket(action, method string, input ISerializable, output IBaseModel) error { + return obsClient.doAction(action, method, "", "", input, output, true, true) +} + +func (obsClient ObsClient) doActionWithBucketV2(action, method, bucketName string, input ISerializable, output IBaseModel) error { + if strings.TrimSpace(bucketName) == "" { + return errors.New("Bucket is empty") + } + return obsClient.doAction(action, method, bucketName, "", input, output, false, true) +} + +func (obsClient ObsClient) doActionWithBucket(action, method, bucketName string, input ISerializable, output IBaseModel) error { + if strings.TrimSpace(bucketName) == "" { + return errors.New("Bucket is empty") + } + return obsClient.doAction(action, method, bucketName, "", input, output, true, true) +} + +func (obsClient ObsClient) doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel) error { + return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, true) +} + +func (obsClient ObsClient) doActionWithBucketAndKeyUnRepeatable(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel) error { + return obsClient._doActionWithBucketAndKey(action, method, bucketName, objectKey, input, output, false) +} + +func (obsClient ObsClient) _doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, repeatable bool) error { + if strings.TrimSpace(bucketName) == "" { + return errors.New("Key is empty") + } + if strings.TrimSpace(objectKey) == "" { + return errors.New("Key is empty") + } + return obsClient.doAction(action, method, bucketName, objectKey, input, output, true, repeatable) +} + +func (obsClient ObsClient) doAction(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, xmlResult bool, repeatable bool) error { + + var resp *http.Response + var respError error + doLog(LEVEL_INFO, "Enter method %s...", action) + start := GetCurrentTimestamp() + + params, headers, data := input.trans(obsClient.conf.signature == SignatureObs) + + if params == nil { + params = make(map[string]string) + } + + if headers == nil { + headers = make(map[string][]string) + } + + switch method { + case HTTP_GET: + resp, respError = obsClient.doHttpGet(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_POST: + resp, respError = obsClient.doHttpPost(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_PUT: + resp, respError = obsClient.doHttpPut(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_DELETE: + resp, respError = obsClient.doHttpDelete(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_HEAD: + resp, respError = obsClient.doHttpHead(bucketName, objectKey, params, headers, data, repeatable) + case HTTP_OPTIONS: + resp, respError = obsClient.doHttpOptions(bucketName, objectKey, params, headers, data, repeatable) + default: + respError = errors.New("Unexpect http method error") + } + if respError == nil && output != nil { + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + } else { + doLog(LEVEL_WARN, "Do http request with error: %v", respError) + } + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } + + return respError +} + +func (obsClient ObsClient) doHttpGet(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHttp(HTTP_GET, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHttpHead(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHttp(HTTP_HEAD, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHttpOptions(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHttp(HTTP_OPTIONS, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHttpDelete(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHttp(HTTP_DELETE, bucketName, objectKey, params, prepareHeaders(headers, false, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHttpPut(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHttp(HTTP_PUT, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHttpPost(bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (*http.Response, error) { + return obsClient.doHttp(HTTP_POST, bucketName, objectKey, params, prepareHeaders(headers, true, obsClient.conf.signature == SignatureObs), data, repeatable) +} + +func (obsClient ObsClient) doHttpWithSignedUrl(action, method string, signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader, output IBaseModel, xmlResult bool) (respError error) { + req, err := http.NewRequest(method, signedUrl, data) + if err != nil { + return err + } + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } + var resp *http.Response + + doLog(LEVEL_INFO, "Do %s with signedUrl %s...", action, signedUrl) + + req.Header = actualSignedRequestHeaders + if value, ok := req.Header[HEADER_HOST_CAMEL]; ok { + req.Host = value[0] + delete(req.Header, HEADER_HOST_CAMEL) + } else if value, ok := req.Header[HEADER_HOST]; ok { + req.Host = value[0] + delete(req.Header, HEADER_HOST) + } + + if value, ok := req.Header[HEADER_CONTENT_LENGTH_CAMEL]; ok { + req.ContentLength = StringToInt64(value[0], -1) + delete(req.Header, HEADER_CONTENT_LENGTH_CAMEL) + } else if value, ok := req.Header[HEADER_CONTENT_LENGTH]; ok { + req.ContentLength = StringToInt64(value[0], -1) + delete(req.Header, HEADER_CONTENT_LENGTH) + } + + req.Header[HEADER_USER_AGENT_CAMEL] = []string{USER_AGENT} + start := GetCurrentTimestamp() + resp, err = obsClient.httpClient.Do(req) + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + } + + var msg interface{} + if err != nil { + respError = err + resp = nil + } else { + doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) + if resp.StatusCode >= 300 { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + msg = resp.Status + resp = nil + } else { + if output != nil { + respError = ParseResponseToBaseModel(resp, output, xmlResult, obsClient.conf.signature == SignatureObs) + } + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } + } + } + + if msg != nil { + doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) + } + + if isDebugLogEnabled() { + doLog(LEVEL_DEBUG, "End method %s, obsclient cost %d ms", action, (GetCurrentTimestamp() - start)) + } + + return +} + +func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params map[string]string, + headers map[string][]string, data interface{}, repeatable bool) (resp *http.Response, respError error) { + + bucketName = strings.TrimSpace(bucketName) + + method = strings.ToUpper(method) + + var redirectUrl string + var requestUrl string + maxRetryCount := obsClient.conf.maxRetryCount + + var _data io.Reader + if data != nil { + if dataStr, ok := data.(string); ok { + doLog(LEVEL_DEBUG, "Do http request with string: %s", dataStr) + headers["Content-Length"] = []string{IntToString(len(dataStr))} + _data = strings.NewReader(dataStr) + } else if dataByte, ok := data.([]byte); ok { + doLog(LEVEL_DEBUG, "Do http request with byte array") + headers["Content-Length"] = []string{IntToString(len(dataByte))} + _data = bytes.NewReader(dataByte) + } else if dataReader, ok := data.(io.Reader); ok { + _data = dataReader + } else { + doLog(LEVEL_WARN, "Data is not a valid io.Reader") + return nil, errors.New("Data is not a valid io.Reader") + } + } + + for i := 0; i <= maxRetryCount; i++ { + if redirectUrl != "" { + parsedRedirectUrl, err := url.Parse(redirectUrl) + if err != nil { + return nil, err + } + requestUrl, _ = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectUrl.Host) + if parsedRequestUrl, _ := url.Parse(requestUrl); parsedRequestUrl.RawQuery != "" && parsedRedirectUrl.RawQuery == "" { + redirectUrl += "?" + parsedRequestUrl.RawQuery + } + requestUrl = redirectUrl + } else { + var err error + requestUrl, err = obsClient.doAuth(method, bucketName, objectKey, params, headers, "") + if err != nil { + return nil, err + } + } + + req, err := http.NewRequest(method, requestUrl, _data) + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } + if err != nil { + return nil, err + } + doLog(LEVEL_DEBUG, "Do request with url [%s] and method [%s]", requestUrl, method) + + if isDebugLogEnabled() { + auth := headers[HEADER_AUTH_CAMEL] + delete(headers, HEADER_AUTH_CAMEL) + doLog(LEVEL_DEBUG, "Request headers: %v", headers) + headers[HEADER_AUTH_CAMEL] = auth + } + + for key, value := range headers { + if key == HEADER_HOST_CAMEL { + req.Host = value[0] + delete(headers, key) + } else if key == HEADER_CONTENT_LENGTH_CAMEL { + req.ContentLength = StringToInt64(value[0], -1) + delete(headers, key) + } else { + req.Header[key] = value + } + } + + req.Header[HEADER_USER_AGENT_CAMEL] = []string{USER_AGENT} + + start := GetCurrentTimestamp() + resp, err = obsClient.httpClient.Do(req) + if isInfoLogEnabled() { + doLog(LEVEL_INFO, "Do http request cost %d ms", (GetCurrentTimestamp() - start)) + } + + var msg interface{} + if err != nil { + msg = err + respError = err + resp = nil + } else { + doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) + if resp.StatusCode < 300 { + break + } else if !repeatable || (resp.StatusCode >= 400 && resp.StatusCode < 500) || resp.StatusCode == 304 { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + break + } else if resp.StatusCode >= 300 && resp.StatusCode < 400 { + if location := resp.Header.Get(HEADER_LOCATION_CAMEL); location != "" { + redirectUrl = location + doLog(LEVEL_WARN, "Redirect request to %s", redirectUrl) + msg = resp.Status + maxRetryCount++ + } else { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + break + } + } else { + msg = resp.Status + } + } + if i != maxRetryCount { + if resp != nil { + resp.Body.Close() + resp = nil + } + if _, ok := headers[HEADER_AUTH_CAMEL]; ok { + delete(headers, HEADER_AUTH_CAMEL) + } + doLog(LEVEL_WARN, "Failed to send request with reason:%v, will try again", msg) + if r, ok := _data.(*strings.Reader); ok { + r.Seek(0, 0) + } else if r, ok := _data.(*bytes.Reader); ok { + r.Seek(0, 0) + } else if r, ok := _data.(*fileReaderWrapper); ok { + fd, err := os.Open(r.filePath) + if err != nil { + return nil, err + } + defer fd.Close() + fileReaderWrapper := &fileReaderWrapper{filePath: r.filePath} + fileReaderWrapper.mark = r.mark + fileReaderWrapper.reader = fd + fileReaderWrapper.totalCount = r.totalCount + _data = fileReaderWrapper + fd.Seek(r.mark, 0) + } else if r, ok := _data.(*readerWrapper); ok { + r.seek(0, 0) + } + time.Sleep(time.Duration(float64(i+2) * rand.Float64() * float64(time.Second))) + } else { + doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) + if resp != nil { + respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) + resp = nil + } + } + } + return +} + +type connDelegate struct { + conn net.Conn + socketTimeout time.Duration + finalTimeout time.Duration +} + +func getConnDelegate(conn net.Conn, socketTimeout int, finalTimeout int) *connDelegate { + return &connDelegate{ + conn: conn, + socketTimeout: time.Second * time.Duration(socketTimeout), + finalTimeout: time.Second * time.Duration(finalTimeout), + } +} + +func (delegate *connDelegate) Read(b []byte) (n int, err error) { + delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout)) + n, err = delegate.conn.Read(b) + delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout)) + return n, err +} + +func (delegate *connDelegate) Write(b []byte) (n int, err error) { + delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout)) + n, err = delegate.conn.Write(b) + finalTimeout := time.Now().Add(delegate.finalTimeout) + delegate.SetWriteDeadline(finalTimeout) + delegate.SetReadDeadline(finalTimeout) + return n, err +} + +func (delegate *connDelegate) Close() error { + return delegate.conn.Close() +} + +func (delegate *connDelegate) LocalAddr() net.Addr { + return delegate.conn.LocalAddr() +} + +func (delegate *connDelegate) RemoteAddr() net.Addr { + return delegate.conn.RemoteAddr() +} + +func (delegate *connDelegate) SetDeadline(t time.Time) error { + return delegate.conn.SetDeadline(t) +} + +func (delegate *connDelegate) SetReadDeadline(t time.Time) error { + return delegate.conn.SetReadDeadline(t) +} + +func (delegate *connDelegate) SetWriteDeadline(t time.Time) error { + return delegate.conn.SetWriteDeadline(t) +} diff --git a/pkg/util/huawei/obs/log.go b/pkg/util/huawei/obs/log.go new file mode 100644 index 0000000000..46420da0d2 --- /dev/null +++ b/pkg/util/huawei/obs/log.go @@ -0,0 +1,277 @@ +package obs + +import ( + "errors" + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" + "sync" +) + +type Level int + +const ( + LEVEL_OFF Level = 500 + LEVEL_ERROR Level = 400 + LEVEL_WARN Level = 300 + LEVEL_INFO Level = 200 + LEVEL_DEBUG Level = 100 +) + +var logLevelMap = map[Level]string{ + LEVEL_OFF: "[OFF]: ", + LEVEL_ERROR: "[ERROR]: ", + LEVEL_WARN: "[WARN]: ", + LEVEL_INFO: "[INFO]: ", + LEVEL_DEBUG: "[DEBUG]: ", +} + +type logConfType struct { + level Level + logToConsole bool + logFullPath string + maxLogSize int64 + backups int +} + +func getDefaultLogConf() logConfType { + return logConfType{ + level: LEVEL_WARN, + logToConsole: false, + logFullPath: "", + maxLogSize: 1024 * 1024 * 30, //30MB + backups: 10, + } +} + +var logConf logConfType + +type loggerWrapper struct { + fullPath string + fd *os.File + ch chan string + wg sync.WaitGroup + queue []string + logger *log.Logger + index int + cacheCount int + closed bool +} + +func (lw *loggerWrapper) doInit() { + lw.queue = make([]string, 0, lw.cacheCount) + lw.logger = log.New(lw.fd, "", 0) + lw.ch = make(chan string, lw.cacheCount) + lw.wg.Add(1) + go lw.doWrite() +} + +func (lw *loggerWrapper) rotate() { + stat, err := lw.fd.Stat() + if err != nil { + lw.fd.Close() + panic(err) + } + if stat.Size() >= logConf.maxLogSize { + lw.fd.Sync() + lw.fd.Close() + if lw.index > logConf.backups { + lw.index = 1 + } + os.Rename(lw.fullPath, lw.fullPath+"."+IntToString(lw.index)) + lw.index += 1 + + fd, err := os.OpenFile(lw.fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + panic(err) + } + lw.fd = fd + lw.logger.SetOutput(lw.fd) + } +} + +func (lw *loggerWrapper) doFlush() { + lw.rotate() + for _, m := range lw.queue { + lw.logger.Println(m) + } + lw.fd.Sync() +} + +func (lw *loggerWrapper) doClose() { + lw.closed = true + close(lw.ch) + lw.wg.Wait() +} + +func (lw *loggerWrapper) doWrite() { + defer lw.wg.Done() + for { + msg, ok := <-lw.ch + if !ok { + lw.doFlush() + lw.fd.Close() + break + } + if len(lw.queue) >= lw.cacheCount { + lw.doFlush() + lw.queue = make([]string, 0, lw.cacheCount) + } + lw.queue = append(lw.queue, msg) + } + +} + +func (lw *loggerWrapper) Printf(format string, v ...interface{}) { + if !lw.closed { + msg := fmt.Sprintf(format, v...) + lw.ch <- msg + } +} + +var consoleLogger *log.Logger +var fileLogger *loggerWrapper +var lock *sync.RWMutex = new(sync.RWMutex) + +func isDebugLogEnabled() bool { + return logConf.level <= LEVEL_DEBUG +} + +func isErrorLogEnabled() bool { + return logConf.level <= LEVEL_ERROR +} + +func isWarnLogEnabled() bool { + return logConf.level <= LEVEL_WARN +} + +func isInfoLogEnabled() bool { + return logConf.level <= LEVEL_INFO +} + +func reset() { + if fileLogger != nil { + fileLogger.doClose() + fileLogger = nil + } + consoleLogger = nil + logConf = getDefaultLogConf() +} + +func InitLog(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool) error { + return InitLogWithCacheCnt(logFullPath, maxLogSize, backups, level, logToConsole, 50) +} + +func InitLogWithCacheCnt(logFullPath string, maxLogSize int64, backups int, level Level, logToConsole bool, cacheCnt int) error { + lock.Lock() + defer lock.Unlock() + if cacheCnt <= 0 { + cacheCnt = 50 + } + reset() + if fullPath := strings.TrimSpace(logFullPath); fullPath != "" { + _fullPath, err := filepath.Abs(fullPath) + if err != nil { + return err + } + + if !strings.HasSuffix(_fullPath, ".log") { + _fullPath += ".log" + } + + stat, err := os.Stat(_fullPath) + if err == nil && stat.IsDir() { + return errors.New(fmt.Sprintf("logFullPath:[%s] is a directory", _fullPath)) + } else if err := os.MkdirAll(filepath.Dir(_fullPath), os.ModePerm); err != nil { + return err + } + + fd, err := os.OpenFile(_fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return err + } + + if stat == nil { + stat, err = os.Stat(_fullPath) + if err != nil { + fd.Close() + return err + } + } + + prefix := stat.Name() + "." + index := 1 + walkFunc := func(path string, info os.FileInfo, err error) error { + if err == nil { + if name := info.Name(); strings.HasPrefix(name, prefix) { + if i := StringToInt(name[len(prefix):], 0); i >= index { + index = i + 1 + } + } + } + return err + } + + if err = filepath.Walk(filepath.Dir(_fullPath), walkFunc); err != nil { + fd.Close() + return err + } + + fileLogger = &loggerWrapper{fullPath: _fullPath, fd: fd, index: index, cacheCount: cacheCnt, closed: false} + fileLogger.doInit() + } + if maxLogSize > 0 { + logConf.maxLogSize = maxLogSize + } + if backups > 0 { + logConf.backups = backups + } + logConf.level = level + if logToConsole { + consoleLogger = log.New(os.Stdout, "", log.LstdFlags) + } + return nil +} + +func CloseLog() { + if logEnabled() { + lock.Lock() + defer lock.Unlock() + reset() + } +} + +func SyncLog() { +} + +func logEnabled() bool { + return consoleLogger != nil || fileLogger != nil +} + +func DoLog(level Level, format string, v ...interface{}) { + doLog(level, format, v) +} + +func doLog(level Level, format string, v ...interface{}) { + if logEnabled() && logConf.level <= level { + msg := fmt.Sprintf(format, v...) + if _, file, line, ok := runtime.Caller(1); ok { + index := strings.LastIndex(file, "/") + if index >= 0 { + file = file[index+1:] + } + msg = fmt.Sprintf("%s:%d|%s", file, line, msg) + } + prefix := logLevelMap[level] + if consoleLogger != nil { + consoleLogger.Printf("%s%s", prefix, msg) + } + if fileLogger != nil { + nowDate := FormatUtcNow("2006-01-02T15:04:05Z") + fileLogger.Printf("%s %s%s", nowDate, prefix, msg) + } + } +} diff --git a/pkg/util/huawei/obs/model.go b/pkg/util/huawei/obs/model.go new file mode 100644 index 0000000000..fc3f5c00fd --- /dev/null +++ b/pkg/util/huawei/obs/model.go @@ -0,0 +1,931 @@ +package obs + +import ( + "encoding/xml" + "io" + "net/http" + "time" +) + +type BaseModel struct { + StatusCode int `xml:"-"` + RequestId string `xml:"RequestId"` + ResponseHeaders map[string][]string `xml:"-"` +} + +type Bucket struct { + XMLName xml.Name `xml:"Bucket"` + Name string `xml:"Name"` + CreationDate time.Time `xml:"CreationDate"` + Location string `xml:"Location"` +} + +type Owner struct { + XMLName xml.Name `xml:"Owner"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName,omitempty"` +} + +type Initiator struct { + XMLName xml.Name `xml:"Initiator"` + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName,omitempty"` +} + +type ListBucketsInput struct { + QueryLocation bool +} + +type ListBucketsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Owner Owner `xml:"Owner"` + Buckets []Bucket `xml:"Buckets>Bucket"` +} + +type bucketLocationObs struct { + XMLName xml.Name `xml:"Location"` + Location string `xml:",chardata"` +} + +type BucketLocation struct { + XMLName xml.Name `xml:"CreateBucketConfiguration"` + Location string `xml:"LocationConstraint,omitempty"` +} + +type CreateBucketInput struct { + BucketLocation + Bucket string `xml:"-"` + ACL AclType `xml:"-"` + StorageClass StorageClassType `xml:"-"` + GrantReadId string `xml:"-"` + GrantWriteId string `xml:"-"` + GrantReadAcpId string `xml:"-"` + GrantWriteAcpId string `xml:"-"` + GrantFullControlId string `xml:"-"` + GrantReadDeliveredId string `xml:"-"` + GrantFullControlDeliveredId string `xml:"-"` + Epid string `xml:"-"` +} + +type BucketStoragePolicy struct { + XMLName xml.Name `xml:"StoragePolicy"` + StorageClass StorageClassType `xml:"DefaultStorageClass"` +} + +type SetBucketStoragePolicyInput struct { + Bucket string `xml:"-"` + BucketStoragePolicy +} + +type getBucketStoragePolicyOutputS3 struct { + BaseModel + BucketStoragePolicy +} + +type GetBucketStoragePolicyOutput struct { + BaseModel + StorageClass string +} + +type bucketStoragePolicyObs struct { + XMLName xml.Name `xml:"StorageClass"` + StorageClass string `xml:",chardata"` +} +type getBucketStoragePolicyOutputObs struct { + BaseModel + bucketStoragePolicyObs +} + +type ListObjsInput struct { + Prefix string + MaxKeys int + Delimiter string + Origin string + RequestHeader string +} + +type ListObjectsInput struct { + ListObjsInput + Bucket string + Marker string +} + +type Content struct { + XMLName xml.Name `xml:"Contents"` + Owner Owner `xml:"Owner"` + ETag string `xml:"ETag"` + Key string `xml:"Key"` + LastModified time.Time `xml:"LastModified"` + Size int64 `xml:"Size"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +type ListObjectsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListBucketResult"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + Marker string `xml:"Marker"` + NextMarker string `xml:"NextMarker"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Contents []Content `xml:"Contents"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + Location string `xml:"-"` +} + +type ListVersionsInput struct { + ListObjsInput + Bucket string + KeyMarker string + VersionIdMarker string +} + +type Version struct { + DeleteMarker + XMLName xml.Name `xml:"Version"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` +} + +type DeleteMarker struct { + XMLName xml.Name `xml:"DeleteMarker"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + IsLatest bool `xml:"IsLatest"` + LastModified time.Time `xml:"LastModified"` + Owner Owner `xml:"Owner"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +type ListVersionsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListVersionsResult"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + KeyMarker string `xml:"KeyMarker"` + NextKeyMarker string `xml:"NextKeyMarker"` + VersionIdMarker string `xml:"VersionIdMarker"` + NextVersionIdMarker string `xml:"NextVersionIdMarker"` + MaxKeys int `xml:"MaxKeys"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Versions []Version `xml:"Version"` + DeleteMarkers []DeleteMarker `xml:"DeleteMarker"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + Location string `xml:"-"` +} + +type ListMultipartUploadsInput struct { + Bucket string + Prefix string + MaxUploads int + Delimiter string + KeyMarker string + UploadIdMarker string +} + +type Upload struct { + XMLName xml.Name `xml:"Upload"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + Initiated time.Time `xml:"Initiated"` + StorageClass StorageClassType `xml:"StorageClass"` + Owner Owner `xml:"Owner"` + Initiator Initiator `xml:"Initiator"` +} + +type ListMultipartUploadsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListMultipartUploadsResult"` + Bucket string `xml:"Bucket"` + KeyMarker string `xml:"KeyMarker"` + NextKeyMarker string `xml:"NextKeyMarker"` + UploadIdMarker string `xml:"UploadIdMarker"` + NextUploadIdMarker string `xml:"NextUploadIdMarker"` + Delimiter string `xml:"Delimiter"` + IsTruncated bool `xml:"IsTruncated"` + MaxUploads int `xml:"MaxUploads"` + Prefix string `xml:"Prefix"` + Uploads []Upload `xml:"Upload"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` +} + +type BucketQuota struct { + XMLName xml.Name `xml:"Quota"` + Quota int64 `xml:"StorageQuota"` +} + +type SetBucketQuotaInput struct { + Bucket string `xml:"-"` + BucketQuota +} + +type GetBucketQuotaOutput struct { + BaseModel + BucketQuota +} + +type GetBucketStorageInfoOutput struct { + BaseModel + XMLName xml.Name `xml:"GetBucketStorageInfoResult"` + Size int64 `xml:"Size"` + ObjectNumber int `xml:"ObjectNumber"` +} + +type getBucketLocationOutputS3 struct { + BaseModel + BucketLocation +} +type getBucketLocationOutputObs struct { + BaseModel + bucketLocationObs +} +type GetBucketLocationOutput struct { + BaseModel + Location string `xml:"-"` +} + +type Grantee struct { + XMLName xml.Name `xml:"Grantee"` + Type GranteeType `xml:"type,attr"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + URI GroupUriType `xml:"URI,omitempty"` +} + +type granteeObs struct { + XMLName xml.Name `xml:"Grantee"` + Type GranteeType `xml:"type,attr"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + Canned string `xml:"Canned,omitempty"` +} + +type Grant struct { + XMLName xml.Name `xml:"Grant"` + Grantee Grantee `xml:"Grantee"` + Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` +} +type grantObs struct { + XMLName xml.Name `xml:"Grant"` + Grantee granteeObs `xml:"Grantee"` + Permission PermissionType `xml:"Permission"` + Delivered bool `xml:"Delivered"` +} + +type AccessControlPolicy struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []Grant `xml:"AccessControlList>Grant"` +} + +type accessControlPolicyObs struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []grantObs `xml:"AccessControlList>Grant"` +} + +type GetBucketAclOutput struct { + BaseModel + AccessControlPolicy +} + +type getBucketAclOutputObs struct { + BaseModel + accessControlPolicyObs +} + +type SetBucketAclInput struct { + Bucket string `xml:"-"` + ACL AclType `xml:"-"` + AccessControlPolicy +} + +type SetBucketPolicyInput struct { + Bucket string + Policy string +} + +type GetBucketPolicyOutput struct { + BaseModel + Policy string `body` +} + +type CorsRule struct { + XMLName xml.Name `xml:"CORSRule"` + ID string `xml:"ID,omitempty"` + AllowedOrigin []string `xml:"AllowedOrigin"` + AllowedMethod []string `xml:"AllowedMethod"` + AllowedHeader []string `xml:"AllowedHeader,omitempty"` + MaxAgeSeconds int `xml:"MaxAgeSeconds"` + ExposeHeader []string `xml:"ExposeHeader,omitempty"` +} + +type BucketCors struct { + XMLName xml.Name `xml:"CORSConfiguration"` + CorsRules []CorsRule `xml:"CORSRule"` +} + +type SetBucketCorsInput struct { + Bucket string `xml:"-"` + BucketCors +} + +type GetBucketCorsOutput struct { + BaseModel + BucketCors +} + +type BucketVersioningConfiguration struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status VersioningStatusType `xml:"Status"` +} + +type SetBucketVersioningInput struct { + Bucket string `xml:"-"` + BucketVersioningConfiguration +} + +type GetBucketVersioningOutput struct { + BaseModel + BucketVersioningConfiguration +} + +type IndexDocument struct { + Suffix string `xml:"Suffix"` +} + +type ErrorDocument struct { + Key string `xml:"Key,omitempty"` +} + +type Condition struct { + XMLName xml.Name `xml:"Condition"` + KeyPrefixEquals string `xml:"KeyPrefixEquals,omitempty"` + HttpErrorCodeReturnedEquals string `xml:"HttpErrorCodeReturnedEquals,omitempty"` +} + +type Redirect struct { + XMLName xml.Name `xml:"Redirect"` + Protocol ProtocolType `xml:"Protocol,omitempty"` + HostName string `xml:"HostName,omitempty"` + ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith,omitempty"` + ReplaceKeyWith string `xml:"ReplaceKeyWith,omitempty"` + HttpRedirectCode string `xml:"HttpRedirectCode,omitempty"` +} + +type RoutingRule struct { + XMLName xml.Name `xml:"RoutingRule"` + Condition Condition `xml:"Condition,omitempty"` + Redirect Redirect `xml:"Redirect"` +} + +type RedirectAllRequestsTo struct { + XMLName xml.Name `xml:"RedirectAllRequestsTo"` + Protocol ProtocolType `xml:"Protocol,omitempty"` + HostName string `xml:"HostName"` +} + +type BucketWebsiteConfiguration struct { + XMLName xml.Name `xml:"WebsiteConfiguration"` + RedirectAllRequestsTo RedirectAllRequestsTo `xml:"RedirectAllRequestsTo,omitempty"` + IndexDocument IndexDocument `xml:"IndexDocument,omitempty"` + ErrorDocument ErrorDocument `xml:"ErrorDocument,omitempty"` + RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` +} + +type SetBucketWebsiteConfigurationInput struct { + Bucket string `xml:"-"` + BucketWebsiteConfiguration +} + +type GetBucketWebsiteConfigurationOutput struct { + BaseModel + BucketWebsiteConfiguration +} + +type GetBucketMetadataInput struct { + Bucket string + Origin string + RequestHeader string +} + +type GetBucketMetadataOutput struct { + BaseModel + StorageClass StorageClassType + Location string + Version string + AllowOrigin string + AllowMethod string + AllowHeader string + MaxAgeSeconds int + ExposeHeader string + Epid string +} + +type BucketLoggingStatus struct { + XMLName xml.Name `xml:"BucketLoggingStatus"` + Agency string `xml:"Agency,omitempty"` + TargetBucket string `xml:"LoggingEnabled>TargetBucket,omitempty"` + TargetPrefix string `xml:"LoggingEnabled>TargetPrefix,omitempty"` + TargetGrants []Grant `xml:"LoggingEnabled>TargetGrants>Grant,omitempty"` +} + +type SetBucketLoggingConfigurationInput struct { + Bucket string `xml:"-"` + BucketLoggingStatus +} + +type GetBucketLoggingConfigurationOutput struct { + BaseModel + BucketLoggingStatus +} + +type Transition struct { + XMLName xml.Name `xml:"Transition"` + Date time.Time `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +type Expiration struct { + XMLName xml.Name `xml:"Expiration"` + Date time.Time `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` +} + +type NoncurrentVersionTransition struct { + XMLName xml.Name `xml:"NoncurrentVersionTransition"` + NoncurrentDays int `xml:"NoncurrentDays"` + StorageClass StorageClassType `xml:"StorageClass"` +} + +type NoncurrentVersionExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration"` + NoncurrentDays int `xml:"NoncurrentDays"` +} + +type LifecycleRule struct { + ID string `xml:"ID,omitempty"` + Prefix string `xml:"Prefix"` + Status RuleStatusType `xml:"Status"` + Transitions []Transition `xml:"Transition,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty"` + NoncurrentVersionTransitions []NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"` + NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"` +} + +type BucketLifecyleConfiguration struct { + XMLName xml.Name `xml:"LifecycleConfiguration"` + LifecycleRules []LifecycleRule `xml:"Rule"` +} + +type SetBucketLifecycleConfigurationInput struct { + Bucket string `xml:"-"` + BucketLifecyleConfiguration +} + +type GetBucketLifecycleConfigurationOutput struct { + BaseModel + BucketLifecyleConfiguration +} + +type Tag struct { + XMLName xml.Name `xml:"Tag"` + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +type BucketTagging struct { + XMLName xml.Name `xml:"Tagging"` + Tags []Tag `xml:"TagSet>Tag"` +} + +type SetBucketTaggingInput struct { + Bucket string `xml:"-"` + BucketTagging +} + +type GetBucketTaggingOutput struct { + BaseModel + BucketTagging +} + +type FilterRule struct { + XMLName xml.Name `xml:"FilterRule"` + Name string `xml:"Name,omitempty"` + Value string `xml:"Value,omitempty"` +} + +type TopicConfiguration struct { + XMLName xml.Name `xml:"TopicConfiguration"` + ID string `xml:"Id,omitempty"` + Topic string `xml:"Topic"` + Events []EventType `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>Object>FilterRule"` +} + +type BucketNotification struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + TopicConfigurations []TopicConfiguration `xml:"TopicConfiguration"` +} + +type SetBucketNotificationInput struct { + Bucket string `xml:"-"` + BucketNotification +} + +type topicConfigurationS3 struct { + XMLName xml.Name `xml:"TopicConfiguration"` + ID string `xml:"Id,omitempty"` + Topic string `xml:"Topic"` + Events []string `xml:"Event"` + FilterRules []FilterRule `xml:"Filter>S3Key>FilterRule"` +} + +type bucketNotificationS3 struct { + XMLName xml.Name `xml:"NotificationConfiguration"` + TopicConfigurations []topicConfigurationS3 `xml:"TopicConfiguration"` +} + +type getBucketNotificationOutputS3 struct { + BaseModel + bucketNotificationS3 +} + +type GetBucketNotificationOutput struct { + BaseModel + BucketNotification +} + +type DeleteObjectInput struct { + Bucket string + Key string + VersionId string +} + +type DeleteObjectOutput struct { + BaseModel + VersionId string + DeleteMarker bool +} + +type ObjectToDelete struct { + XMLName xml.Name `xml:"Object"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` +} + +type DeleteObjectsInput struct { + Bucket string `xml:"-"` + XMLName xml.Name `xml:"Delete"` + Quiet bool `xml:"Quiet,omitempty"` + Objects []ObjectToDelete `xml:"Object"` +} + +type Deleted struct { + XMLName xml.Name `xml:"Deleted"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + DeleteMarker bool `xml:"DeleteMarker"` + DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId"` +} + +type Error struct { + XMLName xml.Name `xml:"Error"` + Key string `xml:"Key"` + VersionId string `xml:"VersionId"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +type DeleteObjectsOutput struct { + BaseModel + XMLName xml.Name `xml:"DeleteResult"` + Deleteds []Deleted `xml:"Deleted"` + Errors []Error `xml:"Error"` +} + +type SetObjectAclInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + VersionId string `xml:"-"` + ACL AclType `xml:"-"` + AccessControlPolicy +} + +type GetObjectAclInput struct { + Bucket string + Key string + VersionId string +} + +type GetObjectAclOutput struct { + BaseModel + VersionId string + AccessControlPolicy +} + +type RestoreObjectInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + VersionId string `xml:"-"` + XMLName xml.Name `xml:"RestoreRequest"` + Days int `xml:"Days"` + Tier RestoreTierType `xml:"GlacierJobParameters>Tier,omitempty"` +} + +type ISseHeader interface { + GetEncryption() string + GetKey() string +} + +type SseKmsHeader struct { + Encryption string + Key string + isObs bool +} + +type SseCHeader struct { + Encryption string + Key string + KeyMD5 string +} + +type GetObjectMetadataInput struct { + Bucket string + Key string + VersionId string + Origin string + RequestHeader string + SseHeader ISseHeader +} + +type GetObjectMetadataOutput struct { + BaseModel + VersionId string + WebsiteRedirectLocation string + Expiration string + Restore string + ObjectType string + NextAppendPosition string + StorageClass StorageClassType + ContentLength int64 + ContentType string + ETag string + AllowOrigin string + AllowHeader string + AllowMethod string + ExposeHeader string + MaxAgeSeconds int + LastModified time.Time + SseHeader ISseHeader + Metadata map[string]string +} + +type GetObjectInput struct { + GetObjectMetadataInput + IfMatch string + IfNoneMatch string + IfUnmodifiedSince time.Time + IfModifiedSince time.Time + RangeStart int64 + RangeEnd int64 + ImageProcess string + ResponseCacheControl string + ResponseContentDisposition string + ResponseContentEncoding string + ResponseContentLanguage string + ResponseContentType string + ResponseExpires string +} + +type GetObjectOutput struct { + GetObjectMetadataOutput + DeleteMarker bool + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + Expires string + Body io.ReadCloser +} + +type ObjectOperationInput struct { + Bucket string + Key string + ACL AclType + GrantReadId string + GrantReadAcpId string + GrantWriteAcpId string + GrantFullControlId string + StorageClass StorageClassType + WebsiteRedirectLocation string + Expires int64 + SseHeader ISseHeader + Metadata map[string]string +} + +type PutObjectBasicInput struct { + ObjectOperationInput + ContentType string + ContentMD5 string + ContentLength int64 +} + +type PutObjectInput struct { + PutObjectBasicInput + Body io.Reader +} + +type PutFileInput struct { + PutObjectBasicInput + SourceFile string +} + +type PutObjectOutput struct { + BaseModel + VersionId string + SseHeader ISseHeader + StorageClass StorageClassType + ETag string +} + +type CopyObjectInput struct { + ObjectOperationInput + CopySourceBucket string + CopySourceKey string + CopySourceVersionId string + CopySourceIfMatch string + CopySourceIfNoneMatch string + CopySourceIfUnmodifiedSince time.Time + CopySourceIfModifiedSince time.Time + SourceSseHeader ISseHeader + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + MetadataDirective MetadataDirectiveType + successActionRedirect string +} + +type CopyObjectOutput struct { + BaseModel + CopySourceVersionId string `xml:"-"` + VersionId string `xml:"-"` + SseHeader ISseHeader `xml:"-"` + XMLName xml.Name `xml:"CopyObjectResult"` + LastModified time.Time `xml:"LastModified"` + ETag string `xml:"ETag"` +} + +type AbortMultipartUploadInput struct { + Bucket string + Key string + UploadId string +} + +type InitiateMultipartUploadInput struct { + ObjectOperationInput + ContentType string +} + +type InitiateMultipartUploadOutput struct { + BaseModel + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + SseHeader ISseHeader +} + +type UploadPartInput struct { + Bucket string + Key string + PartNumber int + UploadId string + ContentMD5 string + SseHeader ISseHeader + Body io.Reader + SourceFile string + Offset int64 + PartSize int64 +} + +type UploadPartOutput struct { + BaseModel + PartNumber int + ETag string + SseHeader ISseHeader +} + +type Part struct { + XMLName xml.Name `xml:"Part"` + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified,omitempty"` + Size int64 `xml:"Size,omitempty"` +} + +type CompleteMultipartUploadInput struct { + Bucket string `xml:"-"` + Key string `xml:"-"` + UploadId string `xml:"-"` + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []Part `xml:"Part"` +} + +type CompleteMultipartUploadOutput struct { + BaseModel + VersionId string `xml:"-"` + SseHeader ISseHeader `xml:"-"` + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +type ListPartsInput struct { + Bucket string + Key string + UploadId string + MaxParts int + PartNumberMarker int +} + +type ListPartsOutput struct { + BaseModel + XMLName xml.Name `xml:"ListPartsResult"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + UploadId string `xml:"UploadId"` + PartNumberMarker int `xml:"PartNumberMarker"` + NextPartNumberMarker int `xml:"NextPartNumberMarker"` + MaxParts int `xml:"MaxParts"` + IsTruncated bool `xml:"IsTruncated"` + StorageClass StorageClassType `xml:"StorageClass"` + Initiator Initiator `xml:"Initiator“` + Owner Owner `xml:"Owner"` + Parts []Part `xml:"Part"` +} + +type CopyPartInput struct { + Bucket string + Key string + UploadId string + PartNumber int + CopySourceBucket string + CopySourceKey string + CopySourceVersionId string + CopySourceRangeStart int64 + CopySourceRangeEnd int64 + SseHeader ISseHeader + SourceSseHeader ISseHeader +} + +type CopyPartOutput struct { + BaseModel + XMLName xml.Name `xml:"CopyPartResult"` + PartNumber int `xml:"-"` + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified"` + SseHeader ISseHeader `xml:"-"` +} + +type CreateSignedUrlInput struct { + Method HttpMethodType + Bucket string + Key string + SubResource SubResourceType + Expires int + Headers map[string]string + QueryParams map[string]string +} + +type CreateSignedUrlOutput struct { + SignedUrl string + ActualSignedRequestHeaders http.Header +} + +type CreateBrowserBasedSignatureInput struct { + Bucket string + Key string + Expires int + FormParams map[string]string +} + +type CreateBrowserBasedSignatureOutput struct { + OriginPolicy string + Policy string + Algorithm string + Credential string + Date string + Signature string +} diff --git a/pkg/util/huawei/obs/temporary.go b/pkg/util/huawei/obs/temporary.go new file mode 100644 index 0000000000..9e457f5121 --- /dev/null +++ b/pkg/util/huawei/obs/temporary.go @@ -0,0 +1,666 @@ +package obs + +import ( + "errors" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" +) + +func (obsClient ObsClient) CreateSignedUrl(input *CreateSignedUrlInput) (output *CreateSignedUrlOutput, err error) { + if input == nil { + return nil, errors.New("CreateSignedUrlInput is nil") + } + + params := make(map[string]string, len(input.QueryParams)) + for key, value := range input.QueryParams { + params[key] = value + } + + if input.SubResource != "" { + params[string(input.SubResource)] = "" + } + + headers := make(map[string][]string, len(input.Headers)) + for key, value := range input.Headers { + headers[key] = []string{value} + } + + if input.Expires <= 0 { + input.Expires = 300 + } + + requestUrl, err := obsClient.doAuthTemporary(string(input.Method), input.Bucket, input.Key, params, headers, int64(input.Expires)) + if err != nil { + return nil, err + } + + output = &CreateSignedUrlOutput{ + SignedUrl: requestUrl, + ActualSignedRequestHeaders: headers, + } + return +} + +func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBasedSignatureInput) (output *CreateBrowserBasedSignatureOutput, err error) { + if input == nil { + return nil, errors.New("CreateBrowserBasedSignatureInput is nil") + } + + params := make(map[string]string, len(input.FormParams)) + for key, value := range input.FormParams { + params[key] = value + } + + date := time.Now().UTC() + shortDate := date.Format(SHORT_DATE_FORMAT) + longDate := date.Format(LONG_DATE_FORMAT) + + credential, _ := getCredential(obsClient.conf.securityProvider.ak, obsClient.conf.region, shortDate) + + if input.Expires <= 0 { + input.Expires = 300 + } + + expiration := date.Add(time.Second * time.Duration(input.Expires)).Format(ISO8601_DATE_FORMAT) + params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX + params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + params[PARAM_DATE_AMZ_CAMEL] = longDate + + if obsClient.conf.securityProvider.securityToken != "" { + params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + } + + matchAnyBucket := true + matchAnyKey := true + count := 5 + if bucket := strings.TrimSpace(input.Bucket); bucket != "" { + params["bucket"] = bucket + matchAnyBucket = false + count-- + } + + if key := strings.TrimSpace(input.Key); key != "" { + params["key"] = key + matchAnyKey = false + count-- + } + + originPolicySlice := make([]string, 0, len(params)+count) + originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"expiration\":\"%s\",", expiration)) + originPolicySlice = append(originPolicySlice, "\"conditions\":[") + for key, value := range params { + if _key := strings.TrimSpace(strings.ToLower(key)); _key != "" { + originPolicySlice = append(originPolicySlice, fmt.Sprintf("{\"%s\":\"%s\"},", _key, value)) + } + } + + if matchAnyBucket { + originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$bucket\", \"\"],") + } + + if matchAnyKey { + originPolicySlice = append(originPolicySlice, "[\"starts-with\", \"$key\", \"\"],") + } + + originPolicySlice = append(originPolicySlice, "]}") + + originPolicy := strings.Join(originPolicySlice, "") + policy := Base64Encode([]byte(originPolicy)) + signature := getSignature(policy, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate) + + output = &CreateBrowserBasedSignatureOutput{ + OriginPolicy: originPolicy, + Policy: policy, + Algorithm: params[PARAM_ALGORITHM_AMZ_CAMEL], + Credential: params[PARAM_CREDENTIAL_AMZ_CAMEL], + Date: params[PARAM_DATE_AMZ_CAMEL], + Signature: signature, + } + return +} + +func (obsClient ObsClient) ListBucketsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListBucketsOutput, err error) { + output = &ListBucketsOutput{} + err = obsClient.doHttpWithSignedUrl("ListBuckets", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) CreateBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("CreateBucket", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("DeleteBucket", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketStoragePolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketStoragePolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStoragePolicyOutput, err error) { + output = &GetBucketStoragePolicyOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketStoragePolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) ListObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListObjectsOutput, err error) { + output = &ListObjectsOutput{} + err = obsClient.doHttpWithSignedUrl("ListObjects", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +func (obsClient ObsClient) ListVersionsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListVersionsOutput, err error) { + output = &ListVersionsOutput{} + err = obsClient.doHttpWithSignedUrl("ListVersions", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if location, ok := output.ResponseHeaders[HEADER_BUCKET_REGION]; ok { + output.Location = location[0] + } + } + return +} + +func (obsClient ObsClient) ListMultipartUploadsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListMultipartUploadsOutput, err error) { + output = &ListMultipartUploadsOutput{} + err = obsClient.doHttpWithSignedUrl("ListMultipartUploads", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketQuota", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketQuotaWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketQuotaOutput, err error) { + output = &GetBucketQuotaOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketQuota", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) HeadBucketWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("HeadBucket", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketMetadataOutput, err error) { + output = &GetBucketMetadataOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetBucketMetadataOutput(output) + } + return +} + +func (obsClient ObsClient) GetBucketStorageInfoWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketStorageInfoOutput, err error) { + output = &GetBucketStorageInfoOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketStorageInfo", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketLocationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLocationOutput, err error) { + output = &GetBucketLocationOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketLocation", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketAclOutput, err error) { + output = &GetBucketAclOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketPolicy", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketPolicyOutput, err error) { + output = &GetBucketPolicyOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketPolicy", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, false) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketPolicyWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("DeleteBucketPolicy", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketCors", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketCorsOutput, err error) { + output = &GetBucketCorsOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketCors", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketCorsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("DeleteBucketCors", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketVersioning", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketVersioningWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketVersioningOutput, err error) { + output = &GetBucketVersioningOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketVersioning", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketWebsiteConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketWebsiteConfigurationOutput, err error) { + output = &GetBucketWebsiteConfigurationOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketWebsiteConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketWebsiteConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("DeleteBucketWebsiteConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketLoggingConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketLoggingConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLoggingConfigurationOutput, err error) { + output = &GetBucketLoggingConfigurationOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketLoggingConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketLifecycleConfiguration", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketLifecycleConfigurationOutput, err error) { + output = &GetBucketLifecycleConfigurationOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketLifecycleConfiguration", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketLifecycleConfigurationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("DeleteBucketLifecycleConfiguration", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketTagging", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketTaggingOutput, err error) { + output = &GetBucketTaggingOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketTagging", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteBucketTaggingWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("DeleteBucketTagging", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetBucketNotification", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetBucketNotificationWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetBucketNotificationOutput, err error) { + output = &GetBucketNotificationOutput{} + err = obsClient.doHttpWithSignedUrl("GetBucketNotification", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) DeleteObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *DeleteObjectOutput, err error) { + output = &DeleteObjectOutput{} + err = obsClient.doHttpWithSignedUrl("DeleteObject", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseDeleteObjectOutput(output) + } + return +} + +func (obsClient ObsClient) DeleteObjectsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *DeleteObjectsOutput, err error) { + output = &DeleteObjectsOutput{} + err = obsClient.doHttpWithSignedUrl("DeleteObjects", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) SetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("SetObjectAcl", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetObjectAclWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectAclOutput, err error) { + output = &GetObjectAclOutput{} + err = obsClient.doHttpWithSignedUrl("GetObjectAcl", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { + output.VersionId = versionId[0] + } + } + return +} + +func (obsClient ObsClient) RestoreObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("RestoreObject", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) GetObjectMetadataWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectMetadataOutput, err error) { + output = &GetObjectMetadataOutput{} + err = obsClient.doHttpWithSignedUrl("GetObjectMetadata", HTTP_HEAD, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetObjectMetadataOutput(output) + } + return +} + +func (obsClient ObsClient) GetObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *GetObjectOutput, err error) { + output = &GetObjectOutput{} + err = obsClient.doHttpWithSignedUrl("GetObject", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseGetObjectOutput(output) + } + return +} + +func (obsClient ObsClient) PutObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *PutObjectOutput, err error) { + output = &PutObjectOutput{} + err = obsClient.doHttpWithSignedUrl("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +func (obsClient ObsClient) PutFileWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, sourceFile string) (output *PutObjectOutput, err error) { + var data io.Reader + sourceFile = strings.TrimSpace(sourceFile) + if sourceFile != "" { + fd, err := os.Open(sourceFile) + if err != nil { + return nil, err + } + defer fd.Close() + + stat, err := fd.Stat() + if err != nil { + return nil, err + } + fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} + fileReaderWrapper.reader = fd + + var contentLength int64 + if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH_CAMEL]; ok { + contentLength = StringToInt64(value[0], -1) + } else if value, ok := actualSignedRequestHeaders[HEADER_CONTENT_LENGTH]; ok { + contentLength = StringToInt64(value[0], -1) + } else { + contentLength = stat.Size() + } + if contentLength > stat.Size() { + return nil, errors.New("ContentLength is larger than fileSize") + } + fileReaderWrapper.totalCount = contentLength + data = fileReaderWrapper + } + + output = &PutObjectOutput{} + err = obsClient.doHttpWithSignedUrl("PutObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParsePutObjectOutput(output) + } + return +} + +func (obsClient ObsClient) CopyObjectWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyObjectOutput, err error) { + output = &CopyObjectOutput{} + err = obsClient.doHttpWithSignedUrl("CopyObject", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseCopyObjectOutput(output) + } + return +} + +func (obsClient ObsClient) AbortMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *BaseModel, err error) { + output = &BaseModel{} + err = obsClient.doHttpWithSignedUrl("AbortMultipartUpload", HTTP_DELETE, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) InitiateMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *InitiateMultipartUploadOutput, err error) { + output = &InitiateMultipartUploadOutput{} + err = obsClient.doHttpWithSignedUrl("InitiateMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseInitiateMultipartUploadOutput(output) + } + return +} + +func (obsClient ObsClient) UploadPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *UploadPartOutput, err error) { + output = &UploadPartOutput{} + err = obsClient.doHttpWithSignedUrl("UploadPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParseUploadPartOutput(output) + } + return +} + +func (obsClient ObsClient) CompleteMultipartUploadWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header, data io.Reader) (output *CompleteMultipartUploadOutput, err error) { + output = &CompleteMultipartUploadOutput{} + err = obsClient.doHttpWithSignedUrl("CompleteMultipartUpload", HTTP_POST, signedUrl, actualSignedRequestHeaders, data, output, true) + if err != nil { + output = nil + } else { + ParseCompleteMultipartUploadOutput(output) + } + return +} + +func (obsClient ObsClient) ListPartsWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *ListPartsOutput, err error) { + output = &ListPartsOutput{} + err = obsClient.doHttpWithSignedUrl("ListParts", HTTP_GET, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } + return +} + +func (obsClient ObsClient) CopyPartWithSignedUrl(signedUrl string, actualSignedRequestHeaders http.Header) (output *CopyPartOutput, err error) { + output = &CopyPartOutput{} + err = obsClient.doHttpWithSignedUrl("CopyPart", HTTP_PUT, signedUrl, actualSignedRequestHeaders, nil, output, true) + if err != nil { + output = nil + } else { + ParseCopyPartOutput(output) + } + return +} diff --git a/pkg/util/huawei/obs/trait.go b/pkg/util/huawei/obs/trait.go new file mode 100644 index 0000000000..d542bbe4ab --- /dev/null +++ b/pkg/util/huawei/obs/trait.go @@ -0,0 +1,701 @@ +package obs + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" +) + +type IReadCloser interface { + setReadCloser(body io.ReadCloser) +} + +func (output *GetObjectOutput) setReadCloser(body io.ReadCloser) { + output.Body = body +} + +func setHeaders(headers map[string][]string, header string, headerValue []string, isObs bool) { + if isObs { + header = HEADER_PREFIX_OBS + header + headers[header] = headerValue + } else { + header = HEADER_PREFIX + header + headers[header] = headerValue + } +} + +func setHeadersNext(headers map[string][]string, header string, headerNext string, headerValue []string, isObs bool) { + if isObs { + headers[header] = headerValue + } else { + headers[headerNext] = headerValue + } +} + +type IBaseModel interface { + setStatusCode(statusCode int) + + setRequestId(requestId string) + + setResponseHeaders(responseHeaders map[string][]string) +} + +type ISerializable interface { + trans(isObs bool) (map[string]string, map[string][]string, interface{}) +} + +type DefaultSerializable struct { + params map[string]string + headers map[string][]string + data interface{} +} + +func (input DefaultSerializable) trans(isObs bool) (map[string]string, map[string][]string, interface{}) { + return input.params, input.headers, input.data +} + +var defaultSerializable = &DefaultSerializable{} + +func newSubResourceSerial(subResource SubResourceType) *DefaultSerializable { + return &DefaultSerializable{map[string]string{string(subResource): ""}, nil, nil} +} + +func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(subResource): ""} + data, _ = ConvertRequestToIoReader(input) + return +} + +func (baseModel *BaseModel) setStatusCode(statusCode int) { + baseModel.StatusCode = statusCode +} + +func (baseModel *BaseModel) setRequestId(requestId string) { + baseModel.RequestId = requestId +} + +func (baseModel *BaseModel) setResponseHeaders(responseHeaders map[string][]string) { + baseModel.ResponseHeaders = responseHeaders +} + +func (input ListBucketsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + headers = make(map[string][]string) + if input.QueryLocation && !isObs { + setHeaders(headers, HEADER_LOCATION_AMZ, []string{"true"}, isObs) + } + return +} + +func (input CreateBucketInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + headers = make(map[string][]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if storageClass == "COLD" { + storageClass = "GLACIER" + } + } + setHeadersNext(headers, HEADER_STORAGE_CLASS_OBS, HEADER_STORAGE_CLASS, []string{storageClass}, isObs) + if epid := string(input.Epid); epid != "" { + setHeaders(headers, HEADER_EPID_HEADERS, []string{epid}, isObs) + } + } + if grantReadId := string(input.GrantReadId); grantReadId != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{grantReadId}, isObs) + } + if grantWriteId := string(input.GrantWriteId); grantWriteId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_OBS, []string{grantWriteId}, isObs) + } + if grantReadAcpId := string(input.GrantReadAcpId); grantReadAcpId != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{grantReadAcpId}, isObs) + } + if grantWriteAcpId := string(input.GrantWriteAcpId); grantWriteAcpId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{grantWriteAcpId}, isObs) + } + if grantFullControlId := string(input.GrantFullControlId); grantFullControlId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{grantFullControlId}, isObs) + } + if grantReadDeliveredId := string(input.GrantReadDeliveredId); grantReadDeliveredId != "" { + setHeaders(headers, HEADER_GRANT_READ_DELIVERED_OBS, []string{grantReadDeliveredId}, true) + } + if grantFullControlDeliveredId := string(input.GrantFullControlDeliveredId); grantFullControlDeliveredId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS, []string{grantFullControlDeliveredId}, true) + } + if location := strings.TrimSpace(input.Location); location != "" { + input.Location = location + data, _ = ConvertRequestToIoReader(input) + } + return +} + +func (input SetBucketStoragePolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + xml := make([]string, 0, 1) + if !isObs { + storageClass := "STANDARD" + if input.StorageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if input.StorageClass == "COLD" { + storageClass = "GLACIER" + } + xml = append(xml, fmt.Sprintf("%s", storageClass)) + } else { + params = map[string]string{string(SubResourceStorageClass): ""} + xml = append(xml, fmt.Sprintf("%s", input.StorageClass)) + } + data = strings.Join(xml, "") + return +} + +func (input ListObjsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = make(map[string]string) + if input.Prefix != "" { + params["prefix"] = input.Prefix + } + if input.Delimiter != "" { + params["delimiter"] = input.Delimiter + } + if input.MaxKeys > 0 { + params["max-keys"] = IntToString(input.MaxKeys) + } + headers = make(map[string][]string) + if origin := strings.TrimSpace(input.Origin); origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{origin} + } + if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader} + } + return +} + +func (input ListObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params, headers, data = input.ListObjsInput.trans(isObs) + if input.Marker != "" { + params["marker"] = input.Marker + } + return +} + +func (input ListVersionsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params, headers, data = input.ListObjsInput.trans(isObs) + params[string(SubResourceVersions)] = "" + if input.KeyMarker != "" { + params["key-marker"] = input.KeyMarker + } + if input.VersionIdMarker != "" { + params["version-id-marker"] = input.VersionIdMarker + } + return +} + +func (input ListMultipartUploadsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceUploads): ""} + if input.Prefix != "" { + params["prefix"] = input.Prefix + } + if input.Delimiter != "" { + params["delimiter"] = input.Delimiter + } + if input.MaxUploads > 0 { + params["max-uploads"] = IntToString(input.MaxUploads) + } + if input.KeyMarker != "" { + params["key-marker"] = input.KeyMarker + } + if input.UploadIdMarker != "" { + params["upload-id-marker"] = input.UploadIdMarker + } + return +} + +func (input SetBucketQuotaInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + return trans(SubResourceQuota, input) +} + +func (input SetBucketAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceAcl): ""} + headers = make(map[string][]string) + + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } else { + data, _ = ConvertAclToXml(input.AccessControlPolicy, false, isObs) + } + return +} + +func (input SetBucketPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourcePolicy): ""} + data = strings.NewReader(input.Policy) + return +} + +func (input SetBucketCorsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceCors): ""} + data, md5, _ := ConvertRequestToIoReaderV2(input) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketVersioningInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + return trans(SubResourceVersioning, input) +} + +func (input SetBucketWebsiteConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceWebsite): ""} + data, _ = ConvertWebsiteConfigurationToXml(input.BucketWebsiteConfiguration, false) + return +} + +func (input GetBucketMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + headers = make(map[string][]string) + if origin := strings.TrimSpace(input.Origin); origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{origin} + } + if requestHeader := strings.TrimSpace(input.RequestHeader); requestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{requestHeader} + } + return +} + +func (input SetBucketLoggingConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceLogging): ""} + data, _ = ConvertLoggingStatusToXml(input.BucketLoggingStatus, false, isObs) + return +} + +func (input SetBucketLifecycleConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceLifecycle): ""} + data, md5 := ConvertLifecyleConfigurationToXml(input.BucketLifecyleConfiguration, true, isObs) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketTaggingInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceTagging): ""} + data, md5, _ := ConvertRequestToIoReaderV2(input) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetBucketNotificationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceNotification): ""} + data, _ = ConvertNotificationToXml(input.BucketNotification, false, isObs) + return +} + +func (input DeleteObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input DeleteObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceDelete): ""} + data, md5, _ := ConvertRequestToIoReaderV2(input) + headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + return +} + +func (input SetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceAcl): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } else { + data, _ = ConvertAclToXml(input.AccessControlPolicy, false, isObs) + } + return +} + +func (input GetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceAcl): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + return +} + +func (input RestoreObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{string(SubResourceRestore): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + if !isObs { + data, _ = ConvertRequestToIoReader(input) + } else { + data = ConverntObsRestoreToXml(input) + } + return +} + +func (header SseKmsHeader) GetEncryption() string { + if header.Encryption != "" { + return header.Encryption + } + if !header.isObs { + return DEFAULT_SSE_KMS_ENCRYPTION + } else { + return DEFAULT_SSE_KMS_ENCRYPTION_OBS + } +} + +func (header SseKmsHeader) GetKey() string { + return header.Key +} + +func (header SseCHeader) GetEncryption() string { + if header.Encryption != "" { + return header.Encryption + } + return DEFAULT_SSE_C_ENCRYPTION +} + +func (header SseCHeader) GetKey() string { + return header.Key +} + +func (header SseCHeader) GetKeyMD5() string { + if header.KeyMD5 != "" { + return header.KeyMD5 + } + + if ret, err := Base64Decode(header.GetKey()); err == nil { + return Base64Md5(ret) + } + return "" +} + +func setSseHeader(headers map[string][]string, sseHeader ISseHeader, sseCOnly bool, isObs bool) { + if sseHeader != nil { + if sseCHeader, ok := sseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_KEY_MD5, []string{sseCHeader.GetEncryption()}, isObs) + } else if sseKmsHeader, ok := sseHeader.(SseKmsHeader); !sseCOnly && ok { + sseKmsHeader.isObs = isObs + setHeaders(headers, HEADER_SSEKMS_ENCRYPTION, []string{sseKmsHeader.GetEncryption()}, isObs) + setHeadersNext(headers, HEADER_SSEKMS_KEY_OBS, HEADER_SSEKMS_KEY_AMZ, []string{sseKmsHeader.GetKey()}, isObs) + } + } +} + +func (input GetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = make(map[string]string) + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + + if input.Origin != "" { + headers[HEADER_ORIGIN_CAMEL] = []string{input.Origin} + } + + if input.RequestHeader != "" { + headers[HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL] = []string{input.RequestHeader} + } + setSseHeader(headers, input.SseHeader, true, isObs) + return +} + +func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params, headers, data = input.GetObjectMetadataInput.trans(isObs) + if input.ResponseCacheControl != "" { + params[PARAM_RESPONSE_CACHE_CONTROL] = input.ResponseCacheControl + } + if input.ResponseContentDisposition != "" { + params[PARAM_RESPONSE_CONTENT_DISPOSITION] = input.ResponseContentDisposition + } + if input.ResponseContentEncoding != "" { + params[PARAM_RESPONSE_CONTENT_ENCODING] = input.ResponseContentEncoding + } + if input.ResponseContentLanguage != "" { + params[PARAM_RESPONSE_CONTENT_LANGUAGE] = input.ResponseContentLanguage + } + if input.ResponseContentType != "" { + params[PARAM_RESPONSE_CONTENT_TYPE] = input.ResponseContentType + } + if input.ResponseExpires != "" { + params[PARAM_RESPONSE_EXPIRES] = input.ResponseExpires + } + if input.ImageProcess != "" { + params[PARAM_IMAGE_PROCESS] = input.ImageProcess + } + if input.RangeStart >= 0 && input.RangeEnd > input.RangeStart { + headers[HEADER_RANGE] = []string{fmt.Sprintf("bytes=%d-%d", input.RangeStart, input.RangeEnd)} + } + + if input.IfMatch != "" { + headers[HEADER_IF_MATCH] = []string{input.IfMatch} + } + if input.IfNoneMatch != "" { + headers[HEADER_IF_NONE_MATCH] = []string{input.IfNoneMatch} + } + if !input.IfModifiedSince.IsZero() { + headers[HEADER_IF_MODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfModifiedSince)} + } + if !input.IfUnmodifiedSince.IsZero() { + headers[HEADER_IF_UNMODIFIED_SINCE] = []string{FormatUtcToRfc1123(input.IfUnmodifiedSince)} + } + return +} + +func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + headers = make(map[string][]string) + params = make(map[string]string) + if acl := string(input.ACL); acl != "" { + setHeaders(headers, HEADER_ACL, []string{acl}, isObs) + } + if GrantReadId := string(input.GrantReadId); GrantReadId != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{GrantReadId}, true) + } + if GrantReadAcpId := string(input.GrantReadAcpId); GrantReadAcpId != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{GrantReadAcpId}, true) + } + if GrantWriteAcpId := string(input.GrantWriteAcpId); GrantWriteAcpId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{GrantWriteAcpId}, true) + } + if GrantFullControlId := string(input.GrantFullControlId); GrantFullControlId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{GrantFullControlId}, true) + } + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == "WARM" { + storageClass = "STANDARD_IA" + } else if storageClass == "COLD" { + storageClass = "GLACIER" + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) + } + if input.WebsiteRedirectLocation != "" { + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + + } + setSseHeader(headers, input.SseHeader, false, isObs) + if input.Expires != 0 { + setHeaders(headers, HEADER_EXPIRES_OBS, []string{Int64ToString(input.Expires)}, true) + } + if input.Metadata != nil { + for key, value := range input.Metadata { + key = strings.TrimSpace(key) + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) + } + } + return +} + +func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params, headers, data = input.ObjectOperationInput.trans(isObs) + + if input.ContentMD5 != "" { + headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} + } + + if input.ContentLength > 0 { + headers[HEADER_CONTENT_LENGTH_CAMEL] = []string{Int64ToString(input.ContentLength)} + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } + + return +} + +func (input PutObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params, headers, data = input.PutObjectBasicInput.trans(isObs) + if input.Body != nil { + data = input.Body + } + return +} + +func (input CopyObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params, headers, data = input.ObjectOperationInput.trans(isObs) + + var copySource string + if input.CopySourceVersionId != "" { + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) + } else { + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) + } + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) + + if directive := string(input.MetadataDirective); directive != "" { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{directive}, isObs) + } + + if input.MetadataDirective == ReplaceMetadata { + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL] = []string{input.CacheControl} + } + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE] = []string{input.ContentLanguage} + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE] = []string{input.ContentType} + } + if input.Expires != "" { + headers[HEADER_EXPIRES] = []string{input.Expires} + } + } + + if input.CopySourceIfMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MATCH, []string{input.CopySourceIfMatch}, isObs) + } + if input.CopySourceIfNoneMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_NONE_MATCH, []string{input.CopySourceIfNoneMatch}, isObs) + } + if !input.CopySourceIfModifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)}, isObs) + } + if !input.CopySourceIfUnmodifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)}, isObs) + } + if input.SourceSseHeader != nil { + if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } + } + if input.successActionRedirect != "" { + headers[HEADER_SUCCESS_ACTION_REDIRECT] = []string{input.successActionRedirect} + } + return +} + +func (input AbortMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{"uploadId": input.UploadId} + return +} + +func (input InitiateMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params, headers, data = input.ObjectOperationInput.trans(isObs) + params[string(SubResourceUploads)] = "" + return +} + +func (input UploadPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} + headers = make(map[string][]string) + setSseHeader(headers, input.SseHeader, true, isObs) + if input.Body != nil { + data = input.Body + } + return +} + +func (input CompleteMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{"uploadId": input.UploadId} + data, _ = ConvertCompleteMultipartUploadInputToXml(input, false) + return +} + +func (input ListPartsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{"uploadId": input.UploadId} + if input.MaxParts > 0 { + params["max-parts"] = IntToString(input.MaxParts) + } + if input.PartNumberMarker > 0 { + params["part-number-marker"] = IntToString(input.PartNumberMarker) + } + return +} + +func (input CopyPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { + params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} + headers = make(map[string][]string, 1) + var copySource string + if input.CopySourceVersionId != "" { + copySource = fmt.Sprintf("%s/%s?versionId=%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false), input.CopySourceVersionId) + } else { + copySource = fmt.Sprintf("%s/%s", input.CopySourceBucket, UrlEncode(input.CopySourceKey, false)) + } + setHeaders(headers, HEADER_COPY_SOURCE, []string{copySource}, isObs) + if input.CopySourceRangeStart >= 0 && input.CopySourceRangeEnd > input.CopySourceRangeStart { + setHeaders(headers, HEADER_COPY_SOURCE_RANGE, []string{fmt.Sprintf("bytes=%d-%d", input.CopySourceRangeStart, input.CopySourceRangeEnd)}, isObs) + } + + setSseHeader(headers, input.SseHeader, true, isObs) + if input.SourceSseHeader != nil { + if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY, []string{sseCHeader.GetKey()}, isObs) + setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) + } + + } + return +} + +type partSlice []Part + +func (parts partSlice) Len() int { + return len(parts) +} + +func (parts partSlice) Less(i, j int) bool { + return parts[i].PartNumber < parts[j].PartNumber +} + +func (parts partSlice) Swap(i, j int) { + parts[i], parts[j] = parts[j], parts[i] +} + +type readerWrapper struct { + reader io.Reader + mark int64 + totalCount int64 + readedCount int64 +} + +func (rw *readerWrapper) seek(offset int64, whence int) (int64, error) { + if r, ok := rw.reader.(*strings.Reader); ok { + return r.Seek(offset, whence) + } else if r, ok := rw.reader.(*bytes.Reader); ok { + return r.Seek(offset, whence) + } else if r, ok := rw.reader.(*os.File); ok { + return r.Seek(offset, whence) + } + return offset, nil +} + +func (rw *readerWrapper) Read(p []byte) (n int, err error) { + if rw.totalCount == 0 { + return 0, io.EOF + } + if rw.totalCount > 0 { + n, err = rw.reader.Read(p) + readedOnce := int64(n) + if remainCount := rw.totalCount - rw.readedCount; remainCount > readedOnce { + rw.readedCount += readedOnce + return n, err + } else { + rw.readedCount += remainCount + return int(remainCount), io.EOF + } + } + return rw.reader.Read(p) +} + +type fileReaderWrapper struct { + readerWrapper + filePath string +} diff --git a/pkg/util/huawei/obs/util.go b/pkg/util/huawei/obs/util.go new file mode 100644 index 0000000000..c191b6c86d --- /dev/null +++ b/pkg/util/huawei/obs/util.go @@ -0,0 +1,394 @@ +package obs + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "fmt" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +var regex = regexp.MustCompile("^[\u4e00-\u9fa5]$") +var ipRegex = regexp.MustCompile("^((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)$") +var v4AuthRegex = regexp.MustCompile("Credential=(.+?),SignedHeaders=(.+?),Signature=.+") +var regionRegex = regexp.MustCompile(".+/\\d+/(.+?)/.+") + +func StringContains(src string, subStr string, subTranscoding string) string { + return strings.Replace(src, subStr, subTranscoding, -1) +} +func XmlTranscoding(src string) string { + srcTmp := StringContains(src, "&", "&") + srcTmp = StringContains(srcTmp, "<", "<") + srcTmp = StringContains(srcTmp, ">", ">") + srcTmp = StringContains(srcTmp, "'", "'") + srcTmp = StringContains(srcTmp, "\"", """) + return srcTmp +} +func StringToInt(value string, def int) int { + ret, err := strconv.Atoi(value) + if err != nil { + ret = def + } + return ret +} + +func StringToInt64(value string, def int64) int64 { + ret, err := strconv.ParseInt(value, 10, 64) + if err != nil { + ret = def + } + return ret +} + +func IntToString(value int) string { + return strconv.Itoa(value) +} + +func Int64ToString(value int64) string { + return strconv.FormatInt(value, 10) +} + +func GetCurrentTimestamp() int64 { + return time.Now().UnixNano() / 1000000 +} + +func FormatUtcNow(format string) string { + return time.Now().UTC().Format(format) +} + +func FormatUtcToRfc1123(t time.Time) string { + ret := t.UTC().Format(time.RFC1123) + return ret[:strings.LastIndex(ret, "UTC")] + "GMT" +} + +func Md5(value []byte) []byte { + m := md5.New() + m.Write(value) + return m.Sum(nil) +} + +func HmacSha1(key, value []byte) []byte { + mac := hmac.New(sha1.New, key) + mac.Write(value) + return mac.Sum(nil) +} + +func HmacSha256(key, value []byte) []byte { + mac := hmac.New(sha256.New, key) + mac.Write(value) + return mac.Sum(nil) +} + +func Base64Encode(value []byte) string { + return base64.StdEncoding.EncodeToString(value) +} + +func Base64Decode(value string) ([]byte, error) { + return base64.StdEncoding.DecodeString(value) +} + +func HexMd5(value []byte) string { + return Hex(Md5(value)) +} + +func Base64Md5(value []byte) string { + return Base64Encode(Md5(value)) +} + +func Sha256Hash(value []byte) []byte { + hash := sha256.New() + hash.Write(value) + return hash.Sum(nil) +} + +func ParseXml(value []byte, result interface{}) error { + if len(value) == 0 { + return nil + } + return xml.Unmarshal(value, result) +} + +func TransToXml(value interface{}) ([]byte, error) { + if value == nil { + return []byte{}, nil + } + return xml.Marshal(value) +} + +func Hex(value []byte) string { + return hex.EncodeToString(value) +} + +func HexSha256(value []byte) string { + return Hex(Sha256Hash(value)) +} + +func UrlDecode(value string) (string, error) { + ret, err := url.QueryUnescape(value) + if err == nil { + return ret, nil + } + return "", err +} + +func IsIP(value string) bool { + return ipRegex.MatchString(value) +} + +func UrlEncode(value string, chineseOnly bool) string { + if chineseOnly { + values := make([]string, 0, len(value)) + for _, val := range value { + _value := string(val) + if regex.MatchString(_value) { + _value = url.QueryEscape(_value) + } + values = append(values, _value) + } + return strings.Join(values, "") + } + return url.QueryEscape(value) +} + +func copyHeaders(m map[string][]string) (ret map[string][]string) { + if m != nil { + ret = make(map[string][]string, len(m)) + for key, values := range m { + _values := make([]string, 0, len(values)) + for _, value := range values { + _values = append(_values, value) + } + ret[strings.ToLower(key)] = _values + } + } else { + ret = make(map[string][]string) + } + + return +} + +func parseHeaders(headers map[string][]string) (signature string, region string, signedHeaders string) { + signature = "v2" + if receviedAuthorization, ok := headers[strings.ToLower(HEADER_AUTH_CAMEL)]; ok && len(receviedAuthorization) > 0 { + if strings.HasPrefix(receviedAuthorization[0], V4_HASH_PREFIX) { + signature = "v4" + matches := v4AuthRegex.FindStringSubmatch(receviedAuthorization[0]) + if len(matches) >= 3 { + region = matches[1] + regions := regionRegex.FindStringSubmatch(region) + if len(regions) >= 2 { + region = regions[1] + } + signedHeaders = matches[2] + } + + } else if strings.HasPrefix(receviedAuthorization[0], V2_HASH_PREFIX) { + signature = "v2" + } + } + return +} + +func getTemporaryKeys() []string { + return []string{ + "Signature", + "signature", + "X-Amz-Signature", + "x-amz-signature", + } +} + +func getIsObs(isTemporary bool, querys []string, headers map[string][]string) bool { + isObs := true + if isTemporary { + for _, value := range querys { + keyPrefix := strings.ToLower(value) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + } else if strings.HasPrefix(value, HEADER_ACCESSS_KEY_AMZ) { + isObs = false + } + } + } else { + for key := range headers { + keyPrefix := strings.ToLower(key) + if strings.HasPrefix(keyPrefix, HEADER_PREFIX) { + isObs = false + break + } + } + } + return isObs +} + +func GetAuthorization(ak, sk, method, bucketName, objectKey, queryUrl string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryUrl, "?") { + queryUrl = queryUrl[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryUrl, "&") + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + params := make(map[string]string) + + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key, _ := UrlDecode(kv[0]) + params[key] = "" + } else if length >= 2 { + key, _ := UrlDecode(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val, _ := UrlDecode(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + isTemporary := false + signature := "v2" + temporaryKeys := getTemporaryKeys() + for _, key := range temporaryKeys { + if _, ok := params[key]; ok { + isTemporary = true + if strings.ToLower(key) == "signature" { + signature = "v2" + } else if strings.ToLower(key) == "x-amz-signature" { + signature = "v4" + } + break + } + } + isObs := getIsObs(isTemporary, querysResult, headers) + headers = copyHeaders(headers) + pathStyle := false + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + pathStyle = true + } + conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + + if isTemporary { + return getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature, conf, params, headers, isObs) + } else { + signature, region, signedHeaders := parseHeaders(headers) + if signature == "v4" { + conf.signature = SignatureV4 + requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestUrl, _ := url.Parse(requestUrl) + headerKeys := strings.Split(signedHeaders, ";") + _headers := make(map[string][]string, len(headerKeys)) + for _, headerKey := range headerKeys { + _headers[headerKey] = headers[headerKey] + } + ret = v4Auth(ak, sk, region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, _headers) + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else if signature == "v2" { + conf.signature = SignatureV2 + _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedUrl, headers, isObs) + v2HashPrefix := V2_HASH_PREFIX + if isObs { + v2HashPrefix = OBS_HASH_PREFIX + } + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + } + return + } + +} + +func getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature string, conf *config, params map[string]string, + headers map[string][]string, isObs bool) (ret map[string]string) { + + if signature == "v4" { + conf.signature = SignatureV4 + + longDate, ok := params[PARAM_DATE_AMZ_CAMEL] + if !ok { + longDate = params[HEADER_DATE_AMZ] + } + shortDate := longDate[:8] + + credential, ok := params[PARAM_CREDENTIAL_AMZ_CAMEL] + if !ok { + credential = params[strings.ToLower(PARAM_CREDENTIAL_AMZ_CAMEL)] + } + + _credential, _ := UrlDecode(credential) + + regions := regionRegex.FindStringSubmatch(_credential) + var region string + if len(regions) >= 2 { + region = regions[1] + } + + _, scope := getCredential(ak, region, shortDate) + + expires, ok := params[PARAM_EXPIRES_AMZ_CAMEL] + if !ok { + expires = params[strings.ToLower(PARAM_EXPIRES_AMZ_CAMEL)] + } + + signedHeaders, ok := params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] + if !ok { + signedHeaders = params[strings.ToLower(PARAM_SIGNEDHEADERS_AMZ_CAMEL)] + } + + algorithm, ok := params[PARAM_ALGORITHM_AMZ_CAMEL] + if !ok { + algorithm = params[strings.ToLower(PARAM_ALGORITHM_AMZ_CAMEL)] + } + + if _, ok := params[PARAM_SIGNATURE_AMZ_CAMEL]; ok { + delete(params, PARAM_SIGNATURE_AMZ_CAMEL) + } else if _, ok := params[strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)]; ok { + delete(params, strings.ToLower(PARAM_SIGNATURE_AMZ_CAMEL)) + } + + ret = make(map[string]string, 6) + ret[PARAM_ALGORITHM_AMZ_CAMEL] = algorithm + ret[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + ret[PARAM_DATE_AMZ_CAMEL] = longDate + ret[PARAM_EXPIRES_AMZ_CAMEL] = expires + ret[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = signedHeaders + + requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestUrl, _ := url.Parse(requestUrl) + stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, strings.Split(signedHeaders, ";"), headers) + ret[PARAM_SIGNATURE_AMZ_CAMEL] = UrlEncode(getSignature(stringToSign, sk, region, shortDate), false) + } else if signature == "v2" { + conf.signature = SignatureV2 + _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + expires, ok := params["Expires"] + if !ok { + expires = params["expires"] + } + headers[HEADER_DATE_CAMEL] = []string{expires} + stringToSign := getV2StringToSign(method, canonicalizedUrl, headers, isObs) + ret = make(map[string]string, 3) + ret["Signature"] = UrlEncode(Base64Encode(HmacSha1([]byte(sk), []byte(stringToSign))), false) + ret["AWSAccessKeyId"] = UrlEncode(ak, false) + ret["Expires"] = UrlEncode(expires, false) + } + + return +} diff --git a/pkg/util/huawei/order.go b/pkg/util/huawei/order.go new file mode 100644 index 0000000000..a6c906b194 --- /dev/null +++ b/pkg/util/huawei/order.go @@ -0,0 +1,74 @@ +package huawei + +import "fmt" + +type SResource struct { + ResourceID string `json:"resourceId"` + CloudServiceType string `json:"cloudServiceType"` + RegionCode string `json:"regionCode"` + ResourceType string `json:"resourceType"` + ResourceSpecCode string `json:"resourceSpecCode"` + Status int64 `json:"status"` +} + +/* +获取订单信息 https://support.huaweicloud.com/api-oce/api_order_00001.html +*/ +func (self *SRegion) GetAllResByOrderId(orderId string) ([]SResource, error) { + domains, err := self.client.getEnabledDomains() + if err != nil { + return nil, err + } + + if domains == nil || len(domains) == 0 { + return nil, fmt.Errorf("GetAllResByOrderId domain is empty") + } else if len(domains) > 1 { + // not supported?? + return nil, fmt.Errorf("GetAllResByOrderId mutliple domain(%d) found", len(domains)) + } + + err = self.ecsClient.Orders.SetDomainId(domains[0].ID) + if err != nil { + return nil, err + } + + resp, err := self.ecsClient.Orders.Get(orderId, nil) + if err != nil { + return nil, err + } + + resources := make([]SResource, 0) + err = resp.Unmarshal(&resources, "resources") + return resources, err +} + +func (self *SRegion) getAllResByType(orderId string, resourceType string) ([]SResource, error) { + res, err := self.GetAllResByOrderId(orderId) + if err != nil { + return nil, err + } + + ret := make([]SResource, 0) + for i := range res { + r := res[i] + if r.ResourceType == resourceType { + ret = append(ret, r) + } + } + + return ret, nil +} + +func (self *SRegion) getAllResIdsByType(orderId string, resourceType string) ([]string, error) { + res, err := self.getAllResByType(orderId, resourceType) + if err != nil { + return nil, err + } + + ids := make([]string, 0) + for _, r := range res { + ids = append(ids, r.ResourceID) + } + + return ids, nil +} diff --git a/pkg/util/huawei/provider/provider.go b/pkg/util/huawei/provider/provider.go index 1abcb9fd94..428d9ef01c 100644 --- a/pkg/util/huawei/provider/provider.go +++ b/pkg/util/huawei/provider/provider.go @@ -14,7 +14,6 @@ type SHuaweiProviderFactory struct { } func (self *SHuaweiProviderFactory) ValidateChangeBandwidth(instanceId string, bandwidth int64) error { - // todo: implement me return nil } @@ -111,7 +110,7 @@ func (self *SHuaweiProvider) GetIRegionById(extId string) (cloudprovider.ICloudR } func (self *SHuaweiProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) { - panic("implement me") + return nil, cloudprovider.ErrNotImplemented } func (self *SHuaweiProvider) GetBalance() (float64, error) { diff --git a/pkg/util/huawei/region.go b/pkg/util/huawei/region.go index d810840458..0af2f4e72d 100644 --- a/pkg/util/huawei/region.go +++ b/pkg/util/huawei/region.go @@ -2,14 +2,18 @@ package huawei import ( "fmt" + "sort" "strconv" + "time" "strings" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/huawei/client" + "yunion.io/x/onecloud/pkg/util/huawei/obs" "yunion.io/x/pkg/util/secrules" ) @@ -22,6 +26,7 @@ type Locales struct { type SRegion struct { client *SHuaweiClient ecsClient *client.Client + obsClient *obs.ObsClient // 对象存储client.请勿直接引用。 Description string `json:"description"` ID string `json:"id"` @@ -35,6 +40,10 @@ type SRegion struct { storageCache *SStoragecache } +func (self *SRegion) GetClient() *SHuaweiClient { + return self.client +} + func (self *SRegion) getECSClient() (*client.Client, error) { var err error @@ -63,6 +72,20 @@ func (self *SRegion) getECSClient() (*client.Client, error) { return self.ecsClient, err } +func (self *SRegion) getOBSClient() (*obs.ObsClient, error) { + if self.obsClient == nil { + endpoint := fmt.Sprintf("obs.%s.myhuaweicloud.com", self.GetId()) + obsClient, err := obs.New(self.client.accessKey, self.client.secret, endpoint) + if err != nil { + return nil, err + } + + self.obsClient = obsClient + } + + return self.obsClient, nil +} + func (self *SRegion) fetchZones() error { zones := make([]SZone, 0) err := DoList(self.ecsClient.Zones.List, nil, &zones) @@ -176,12 +199,13 @@ func (self *SRegion) fetchInfrastructure() error { } for i := 0; i < len(self.ivpcs); i += 1 { + vpc := self.ivpcs[i].(*SVpc) + wire := SWire{region: self, vpc: vpc} + vpc.addWire(&wire) + for j := 0; j < len(self.izones); j += 1 { zone := self.izones[j].(*SZone) - vpc := self.ivpcs[i].(*SVpc) - wire := SWire{zone: zone, vpc: vpc} zone.addWire(&wire) - vpc.addWire(&wire) } } return nil @@ -295,40 +319,55 @@ func (self *SRegion) GetIEipById(eipId string) (cloudprovider.ICloudEIP, error) return &eip, err } +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0060595555.html func (self *SRegion) DeleteSecurityGroup(vpcId, secgroupId string) error { - // todo: implement me - return nil + return DoDelete(self.ecsClient.SecurityGroups.Delete, secgroupId, nil, nil) } func (self *SRegion) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) { - //if len(secgroupId) > 0 { - // _, total, err := self.GetSecurityGroups("", []string{secgroupId}, 0, 1) - // if err != nil { - // return "", err - // } - // if total == 0 { - // secgroupId = "" - // } - //} - //if len(secgroupId) == 0 { - // extID, err := self.CreateSecurityGroup(vpcId, name, desc) - // if err != nil { - // return "", err - // } - // secgroupId = extID - //} - //return secgroupId, self.syncSecgroupRules(secgroupId, rules) - // todo: implement me - return "", nil + if len(secgroupId) > 0 { + _, err := self.GetSecurityGroupDetails(secgroupId) + if err == cloudprovider.ErrNotSupported { + secgroupId = "" + } else if err != nil { + return "", err + } + } + + if len(secgroupId) == 0 { + extID, err := self.CreateSecurityGroup(vpcId, name, desc) + if err != nil { + return "", err + } + secgroupId = extID + } + + return secgroupId, self.syncSecgroupRules(secgroupId, rules) } +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090608.html func (self *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) { - // todo: implement me - return self.GetIVpcById("") + params := jsonutils.NewDict() + vpcObj := jsonutils.NewDict() + vpcObj.Add(jsonutils.NewString(name), "name") + vpcObj.Add(jsonutils.NewString(cidr), "cidr") + params.Add(vpcObj, "vpc") + + vpc := SVpc{} + err := DoCreate(self.ecsClient.Vpcs.Create, params, &vpc) + vpc.region = self + return &vpc, err } -func (self *SRegion) CreateEIP(name string, bwMbps int, chargeType string) (cloudprovider.ICloudEIP, error) { - // todo: implement me +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090596.html +// size: 1Mbit/s~2000Mbit/s +// bgpType: 5_telcom，5_union，5_bgp，5_sbgp. +// 东北-大连：5_telcom、5_union +// 华南-广州：5_sbgp +// 华东-上海二：5_sbgp +// 华北-北京一：5_bgp、5_sbgp +// 亚太-香港：5_bgp +func (self *SRegion) CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (cloudprovider.ICloudEIP, error) { var ctype TInternetChargeType switch chargeType { case models.EIP_CHARGE_TYPE_BY_TRAFFIC: @@ -337,7 +376,27 @@ func (self *SRegion) CreateEIP(name string, bwMbps int, chargeType string) (clou ctype = InternetChargeByBandwidth } - eip, err := self.AllocateEIP(bwMbps, ctype) + // todo: 如何避免hardcode。集成到cloudmeta服务中？ + if len(bgpType) == 0 { + switch self.GetId() { + case "cn-north-1", "cn-east-2", "cn-south-1": + bgpType = "5_sbgp" + case "cn-northeast-1": + bgpType = "5_telcom" + case "ap-southeast-1", "ap-southeast-2", "eu-west-0": + bgpType = "5_bgp" + default: + bgpType = "" + } + } + + eip, err := self.AllocateEIP(name, bwMbps, ctype, bgpType) + eip.region = self + if err != nil { + return nil, err + } + + err = cloudprovider.WaitStatus(eip, models.EIP_STATUS_READY, 5*time.Second, 60*time.Second) return eip, err } @@ -442,3 +501,147 @@ func (self *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, er func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_HUAWEI } + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090615.html +// 目前desc字段并没有用到 +func (self *SRegion) CreateSecurityGroup(vpcId string, name string, desc string) (string, error) { + // 华为不允许创建名称为default的安全组 + if strings.ToLower(name) == "default" { + name = fmt.Sprintf("%s-%s", vpcId, name) + } + + params := jsonutils.NewDict() + secgroupObj := jsonutils.NewDict() + secgroupObj.Add(jsonutils.NewString(name), "name") + if len(vpcId) > 0 { + secgroupObj.Add(jsonutils.NewString(vpcId), "vpc_id") + } + params.Add(secgroupObj, "security_group") + + secgroup := SSecurityGroup{} + err := DoCreate(self.ecsClient.SecurityGroups.Create, params, &secgroup) + return secgroup.GetId(), err +} + +func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.SecurityRule) error { + var DeleteRules []secrules.SecurityRule + var AddRules []secrules.SecurityRule + + if secgroup, err := self.GetSecurityGroupDetails(secgroupId); err != nil { + return err + } else { + remoteRules, err := secgroup.GetRulesWithExtId() + if err != nil { + return err + } + + sort.Sort(secrules.SecurityRuleSet(rules)) + sort.Sort(secrules.SecurityRuleSet(remoteRules)) + + i, j := 0, 0 + for i < len(rules) || j < len(remoteRules) { + if i < len(rules) && j < len(remoteRules) { + permissionStr := remoteRules[j].String() + ruleStr := rules[i].String() + cmp := strings.Compare(permissionStr, ruleStr) + if cmp == 0 { + // DeleteRules = append(DeleteRules, remoteRules[j]) + // AddRules = append(AddRules, rules[i]) + i += 1 + j += 1 + } else if cmp > 0 { + DeleteRules = append(DeleteRules, remoteRules[j]) + j += 1 + } else { + AddRules = append(AddRules, rules[i]) + i += 1 + } + } else if i >= len(rules) { + DeleteRules = append(DeleteRules, remoteRules[j]) + j += 1 + } else if j >= len(remoteRules) { + AddRules = append(AddRules, rules[i]) + i += 1 + } + } + } + + for _, r := range DeleteRules { + // r.Description 实际存储的是ruleId + if err := self.delSecurityGroupRule(r.Description); err != nil { + log.Errorf("delSecurityGroupRule %v error: %s", r, err.Error()) + return err + } + } + + for _, r := range AddRules { + if err := self.addSecurityGroupRules(secgroupId, &r); err != nil { + log.Errorf("addSecurityGroupRule %v error: %s", r, err.Error()) + return err + } + } + + return nil +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0087467071.html +func (self *SRegion) delSecurityGroupRule(secGrpRuleId string) error { + return DoDelete(self.ecsClient.SecurityGroupRules.Delete, secGrpRuleId, nil, nil) +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0087451723.html +// icmp port对应关系：https://support.huaweicloud.com/api-vpc/zh-cn_topic_0024109590.html +func (self *SRegion) addSecurityGroupRules(secGrpId string, rule *secrules.SecurityRule) error { + direction := "" + if rule.Direction == secrules.SecurityRuleIngress { + direction = "ingress" + } else { + direction = "egress" + } + + protocal := rule.Protocol + if rule.Protocol == secrules.PROTO_ANY { + protocal = "" + } + + if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + portStr := fmt.Sprintf("%d", port) + err := self.addSecurityGroupRule(secGrpId, direction, portStr, portStr, protocal, rule.IPNet.String()) + if err != nil { + return err + } + } + } else { + portStart := fmt.Sprintf("%d", rule.PortStart) + portEnd := fmt.Sprintf("%d", rule.PortEnd) + err := self.addSecurityGroupRule(secGrpId, direction, portStart, portEnd, protocal, rule.IPNet.String()) + if err != nil { + return err + } + } + + return nil +} + +func (self *SRegion) addSecurityGroupRule(secGrpId, direction, portStart, portEnd, protocol, ipNet string) error { + params := jsonutils.NewDict() + secgroupObj := jsonutils.NewDict() + secgroupObj.Add(jsonutils.NewString(secGrpId), "security_group_id") + secgroupObj.Add(jsonutils.NewString(direction), "direction") + secgroupObj.Add(jsonutils.NewString(ipNet), "remote_ip_prefix") + secgroupObj.Add(jsonutils.NewString("IPV4"), "ethertype") + if len(portStart) > 0 && portStart != "0" { + secgroupObj.Add(jsonutils.NewString(portStart), "port_range_min") + } + if len(portEnd) > 0 && portEnd != "0" { + secgroupObj.Add(jsonutils.NewString(portEnd), "port_range_max") + } + if len(protocol) > 0 { + secgroupObj.Add(jsonutils.NewString(protocol), "protocol") + } + params.Add(secgroupObj, "security_group_rule") + + rule := SecurityGroupRule{} + return DoCreate(self.ecsClient.SecurityGroupRules.Create, params, &rule) +} diff --git a/pkg/util/huawei/securitygroup.go b/pkg/util/huawei/securitygroup.go index 8349b67958..5638e7df02 100644 --- a/pkg/util/huawei/securitygroup.go +++ b/pkg/util/huawei/securitygroup.go @@ -44,7 +44,8 @@ type SecurityGroupRuleDetail struct { // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090615.html type SSecurityGroup struct { - vpc *SVpc + region *SRegion + vpc *SVpc // 安全组对应的vpc可能为空 ID string `json:"id"` Name string `json:"name"` @@ -54,6 +55,21 @@ type SSecurityGroup struct { SecurityGroupRules []SecurityGroupRule `json:"security_group_rules"` } +// 判断是否兼容云端安全组规则 +func compatibleSecurityGroupRule(r SecurityGroupRule) bool { + // 忽略了源地址是安全组的规则 + if len(r.RemoteGroupID) > 0 { + return false + } + + // 忽略IPV6 + if r.Ethertype == "IPv6" { + return false + } + + return true +} + func (self *SSecurityGroup) GetId() string { return self.ID } @@ -83,7 +99,7 @@ func (self *SSecurityGroup) GetStatus() string { } func (self *SSecurityGroup) Refresh() error { - if new, err := self.vpc.region.GetSecurityGroupDetails(self.GetId()); err != nil { + if new, err := self.region.GetSecurityGroupDetails(self.GetId()); err != nil { return err } else { return jsonutils.Update(self, new) @@ -103,15 +119,15 @@ func (self *SSecurityGroup) GetDescription() string { return self.Description } +// todo: 这里需要优化查询太多了 func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { rules := make([]secrules.SecurityRule, 0) for _, r := range self.SecurityGroupRules { - // 忽略了源地址是安全组的规则 - if len(r.RemoteGroupID) > 0 { + if !compatibleSecurityGroupRule(r) { continue } - rule, err := self.GetSecurityRule(r.ID) + rule, err := self.GetSecurityRule(r.ID, false) if err != nil { return rules, err } @@ -122,9 +138,28 @@ func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { return rules, nil } -func (self *SSecurityGroup) GetSecurityRule(ruleId string) (secrules.SecurityRule, error) { +func (self *SSecurityGroup) GetRulesWithExtId() ([]secrules.SecurityRule, error) { + rules := make([]secrules.SecurityRule, 0) + for _, r := range self.SecurityGroupRules { + if !compatibleSecurityGroupRule(r) { + continue + } + + rule, err := self.GetSecurityRule(r.ID, true) + if err != nil { + return rules, err + } + + rules = append(rules, rule) + } + + return rules, nil +} + +// withRuleId. +func (self *SSecurityGroup) GetSecurityRule(ruleId string, withRuleId bool) (secrules.SecurityRule, error) { remoteRule := SecurityGroupRuleDetail{} - err := DoGet(self.vpc.region.ecsClient.SecurityGroupRules.Get, ruleId, nil, &remoteRule) + err := DoGet(self.region.ecsClient.SecurityGroupRules.Get, ruleId, nil, &remoteRule) if err != nil { return secrules.SecurityRule{}, err } @@ -136,17 +171,26 @@ func (self *SSecurityGroup) GetSecurityRule(ruleId string) (secrules.SecurityRul direction = secrules.SecurityRuleEgress } - protocol := "any" + protocol := secrules.PROTO_ANY if remoteRule.Protocol != "" { protocol = remoteRule.Protocol } - // todo: 没考虑ipv6。可能报错 ipNet := &net.IPNet{} if len(remoteRule.RemoteIPPrefix) > 0 { _, ipNet, err = net.ParseCIDR(remoteRule.RemoteIPPrefix) + } else { + _, ipNet, err = net.ParseCIDR("0.0.0.0/0") } + // withRuleId.将ruleId附加到description字段。该hook有特殊目的，仅在同步安全组时使用。 + desc := "" + if withRuleId { + desc = ruleId + } else { + desc = remoteRule.Description + } + // todo: icmp 可能不兼容 rule := secrules.SecurityRule{ Priority: 0, Action: secrules.SecurityRuleAllow, @@ -156,7 +200,7 @@ func (self *SSecurityGroup) GetSecurityRule(ruleId string) (secrules.SecurityRul PortStart: int(remoteRule.PortRangeMin), PortEnd: int(remoteRule.PortRangeMax), Ports: nil, - Description: remoteRule.Description, + Description: desc, } return rule, err } @@ -164,6 +208,15 @@ func (self *SSecurityGroup) GetSecurityRule(ruleId string) (secrules.SecurityRul func (self *SRegion) GetSecurityGroupDetails(secGroupId string) (*SSecurityGroup, error) { securitygroup := SSecurityGroup{} err := DoGet(self.ecsClient.SecurityGroups.Get, secGroupId, nil, &securitygroup) + if err != nil { + return nil, err + } + + securitygroup.region = self + if len(securitygroup.VpcID) > 0 && securitygroup.VpcID != "default" { + securitygroup.vpc, err = self.getVpc(securitygroup.VpcID) + } + return &securitygroup, err } @@ -180,5 +233,33 @@ func (self *SRegion) GetSecurityGroups(vpcId string, limit int, marker string) ( querys["limit"] = strconv.Itoa(limit) securitygroups := make([]SSecurityGroup, 0) err := DoList(self.ecsClient.SecurityGroups.List, querys, &securitygroups) + if err != nil { + return nil, 0, err + } + + vpcCache := map[string]*SVpc{} + for i := range securitygroups { + securitygroup := &securitygroups[i] + securitygroup.region = self + // 未绑定VPC的安全组 + // todo:确认 vpc_id = default的安全组有什么含义？ + if len(securitygroup.VpcID) == 0 || securitygroup.VpcID == "default" { + continue + } + + if vpc, exists := vpcCache[securitygroup.VpcID]; exists { + securitygroup.vpc = vpc + } else { + vpc, err := self.getVpc(securitygroup.VpcID) + if err != nil { + return nil, 0, err + } + + vpcCache[securitygroup.VpcID] = vpc + securitygroup.vpc = vpc + } + + } + return securitygroups, len(securitygroups), err } diff --git a/pkg/util/huawei/shell/business.go b/pkg/util/huawei/shell/business.go new file mode 100644 index 0000000000..42c3bc3a76 --- /dev/null +++ b/pkg/util/huawei/shell/business.go @@ -0,0 +1,19 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type AccountBalanceOptions struct { + } + shellutils.R(&AccountBalanceOptions{}, "balance", "Get account balance", func(cli *huawei.SRegion, args *AccountBalanceOptions) error { + result, err := cli.GetClient().QueryAccountBalance() + if err != nil { + return err + } + printObject(result) + return nil + }) +} diff --git a/pkg/util/huawei/shell/disk.go b/pkg/util/huawei/shell/disk.go new file mode 100644 index 0000000000..0dc680ebd6 --- /dev/null +++ b/pkg/util/huawei/shell/disk.go @@ -0,0 +1,33 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type DiskListOptions struct { + Zone string `help:"Zone ID"` + Offset int `help:"List offset"` + Limit int `help:"List limit"` + } + shellutils.R(&DiskListOptions{}, "disk-list", "List disks", func(cli *huawei.SRegion, args *DiskListOptions) error { + disks, total, e := cli.GetDisks(args.Zone, args.Offset, args.Limit) + if e != nil { + return e + } + printList(disks, total, args.Offset, args.Limit, []string{}) + return nil + }) + + type DiskDeleteOptions struct { + ID string `help:"Disk ID"` + } + shellutils.R(&DiskDeleteOptions{}, "disk-delete", "List disks", func(cli *huawei.SRegion, args *DiskDeleteOptions) error { + e := cli.DeleteDisk(args.ID) + if e != nil { + return e + } + return nil + }) +} diff --git a/pkg/util/huawei/shell/doc.go b/pkg/util/huawei/shell/doc.go new file mode 100644 index 0000000000..0e6cc4e131 --- /dev/null +++ b/pkg/util/huawei/shell/doc.go @@ -0,0 +1 @@ +package shell // import "yunion.io/x/onecloud/pkg/util/huawei/shell" diff --git a/pkg/util/huawei/shell/eip.go b/pkg/util/huawei/shell/eip.go new file mode 100644 index 0000000000..a00694312b --- /dev/null +++ b/pkg/util/huawei/shell/eip.go @@ -0,0 +1,56 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type EipListOptions struct { + Marker string `help:"marker"` + Limit int `help:"List limit"` + } + shellutils.R(&EipListOptions{}, "eip-list", "List eips", func(cli *huawei.SRegion, args *EipListOptions) error { + eips, total, e := cli.GetEips(args.Marker, args.Limit) + if e != nil { + return e + } + printList(eips, total, 0, args.Limit, []string{}) + return nil + }) + + type EipAllocateOptions struct { + Name string `help:"eip name"` + BW int `help:"Bandwidth limit in Mbps"` + BGP string `help:"bgp type" choices:"5_telcom|5_union|5_bgp|5_sbgp"` + } + shellutils.R(&EipAllocateOptions{}, "eip-create", "Allocate an EIP", func(cli *huawei.SRegion, args *EipAllocateOptions) error { + eip, err := cli.AllocateEIP(args.Name, args.BW, huawei.InternetChargeByTraffic, args.BGP) + if err != nil { + return err + } + printObject(eip) + return nil + }) + + type EipReleaseOptions struct { + ID string `help:"EIP allocation ID"` + } + shellutils.R(&EipReleaseOptions{}, "eip-delete", "Release an EIP", func(cli *huawei.SRegion, args *EipReleaseOptions) error { + err := cli.DeallocateEIP(args.ID) + return err + }) + + type EipAssociateOptions struct { + ID string `help:"EIP allocation ID"` + INSTANCE string `help:"Instance ID"` + } + shellutils.R(&EipAssociateOptions{}, "eip-associate", "Associate an EIP", func(cli *huawei.SRegion, args *EipAssociateOptions) error { + err := cli.AssociateEip(args.ID, args.INSTANCE) + return err + }) + shellutils.R(&EipAssociateOptions{}, "eip-dissociate", "Dissociate an EIP", func(cli *huawei.SRegion, args *EipAssociateOptions) error { + err := cli.DissociateEip(args.ID, args.INSTANCE) + return err + }) +} diff --git a/pkg/util/huawei/shell/image.go b/pkg/util/huawei/shell/image.go new file mode 100644 index 0000000000..23df09eda8 --- /dev/null +++ b/pkg/util/huawei/shell/image.go @@ -0,0 +1,32 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ImageListOptions struct { + Status string `help:"image status type" choices:"queued|saving|deleted|killed|active"` + Owner string `help:"Owner type" choices:"gold|private|shared"` + Id []string `help:"Image ID"` + Name string `help:"image name"` + Marker string `help:"marker"` + Limit int `help:"page Limit"` + } + shellutils.R(&ImageListOptions{}, "image-list", "List images", func(cli *huawei.SRegion, args *ImageListOptions) error { + images, total, e := cli.GetImages(args.Status, huawei.ImageOwnerType(args.Owner), args.Name, args.Limit, args.Marker) + if e != nil { + return e + } + printList(images, total, 0, args.Limit, []string{}) + return nil + }) + + type ImageDeleteOptions struct { + ID string `help:"ID or Name to delete"` + } + shellutils.R(&ImageDeleteOptions{}, "image-delete", "Delete image", func(cli *huawei.SRegion, args *ImageDeleteOptions) error { + return cli.DeleteImage(args.ID) + }) +} diff --git a/pkg/util/huawei/shell/instance.go b/pkg/util/huawei/shell/instance.go new file mode 100644 index 0000000000..ec48ce9401 --- /dev/null +++ b/pkg/util/huawei/shell/instance.go @@ -0,0 +1,140 @@ +package shell + +import ( + "context" + + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type InstanceListOptions struct { + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + shellutils.R(&InstanceListOptions{}, "instance-list", "List intances", func(cli *huawei.SRegion, args *InstanceListOptions) error { + instances, total, e := cli.GetInstances(args.Offset, args.Limit) + if e != nil { + return e + } + printList(instances, total, args.Offset, args.Limit, []string{}) + return nil + }) + + type InstanceDiskOperationOptions struct { + ID string `help:"instance ID"` + DISK string `help:"disk ID"` + } + + type InstanceDiskAttachOptions struct { + ID string `help:"instance ID"` + DISK string `help:"disk ID"` + DEVICE string `help:"disk device name. eg. /dev/sdb"` + } + + shellutils.R(&InstanceDiskAttachOptions{}, "instance-attach-disk", "Attach a disk to instance", func(cli *huawei.SRegion, args *InstanceDiskAttachOptions) error { + err := cli.AttachDisk(args.ID, args.DISK, args.DEVICE) + if err != nil { + return err + } + return nil + }) + + shellutils.R(&InstanceDiskOperationOptions{}, "instance-detach-disk", "Detach a disk to instance", func(cli *huawei.SRegion, args *InstanceDiskOperationOptions) error { + err := cli.DetachDisk(args.ID, args.DISK) + if err != nil { + return err + } + return nil + }) + + type InstanceOperationOptions struct { + ID string `help:"instance ID"` + } + shellutils.R(&InstanceOperationOptions{}, "instance-start", "Start a instance", func(cli *huawei.SRegion, args *InstanceOperationOptions) error { + err := cli.StartVM(args.ID) + if err != nil { + return err + } + return nil + }) + + type InstanceStopOptions struct { + ID string `help:"instance ID"` + Force bool `help:"Force stop instance"` + } + shellutils.R(&InstanceStopOptions{}, "instance-stop", "Stop a instance", func(cli *huawei.SRegion, args *InstanceStopOptions) error { + err := cli.StopVM(args.ID, args.Force) + if err != nil { + return err + } + return nil + }) + shellutils.R(&InstanceOperationOptions{}, "instance-delete", "Delete a instance", func(cli *huawei.SRegion, args *InstanceOperationOptions) error { + err := cli.DeleteVM(args.ID) + if err != nil { + return err + } + return nil + }) + + /* + server-change-config 更改系统配置 + server-reset + */ + type InstanceDeployOptions struct { + ID string `help:"instance ID"` + Name string `help:"new instance name"` + Hostname string `help:"new hostname"` + Keypair string `help:"Keypair Name"` + DeleteKeypair bool `help:"Remove SSH keypair"` + Password string `help:"new password"` + // ResetPassword bool `help:"Force reset password"` + Description string `help:"new instances description"` + } + + shellutils.R(&InstanceDeployOptions{}, "instance-deploy", "Deploy keypair/password to a stopped virtual server", func(cli *huawei.SRegion, args *InstanceDeployOptions) error { + err := cli.DeployVM(args.ID, args.Name, args.Password, args.Keypair, args.DeleteKeypair, args.Description) + if err != nil { + return err + } + return nil + }) + + type InstanceRebuildRootOptions struct { + ID string `help:"instance ID"` + Image string `help:"Image ID"` + Password string `help:"admin password"` + PublicKey string `help:"public key name"` + } + + shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Reinstall virtual server system image", func(cli *huawei.SRegion, args *InstanceRebuildRootOptions) error { + ctx := context.Background() + err := cli.ChangeRoot(ctx, args.ID, args.Image, args.Password, args.PublicKey) + if err != nil { + return err + } + // fmt.Printf("New diskID is %s", diskID) + return nil + }) + + type InstanceChangeConfigOptions struct { + ID string `help:"instance ID"` + InstanceTypeId string `help:"instance type"` + Disk []int `help:"Data disk sizes int GB"` + } + + shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Deploy keypair/password to a stopped virtual server", func(cli *huawei.SRegion, args *InstanceChangeConfigOptions) error { + instance, e := cli.GetInstanceByID(args.ID) + if e != nil { + return e + } + + // todo : add create disks + err := cli.ChangeVMConfig2(instance.GetId(), args.ID, args.InstanceTypeId, nil) + if err != nil { + return err + } + return nil + }) +} diff --git a/pkg/util/huawei/shell/instancetype.go b/pkg/util/huawei/shell/instancetype.go new file mode 100644 index 0000000000..b8e6d5ad44 --- /dev/null +++ b/pkg/util/huawei/shell/instancetype.go @@ -0,0 +1,22 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type InstanceMatchOptions struct { + CPU int `help:"CPU count"` + MEM int `help:"Memory in MB"` + Zone string `help:"Test in zone"` + } + shellutils.R(&InstanceMatchOptions{}, "instance-type-select", "Select matching instance types", func(cli *huawei.SRegion, args *InstanceMatchOptions) error { + instanceTypes, e := cli.GetMatchInstanceTypes(args.CPU, args.MEM, args.Zone) + if e != nil { + return e + } + printList(instanceTypes, 0, 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/util/huawei/shell/keypair.go b/pkg/util/huawei/shell/keypair.go new file mode 100644 index 0000000000..94e06df2de --- /dev/null +++ b/pkg/util/huawei/shell/keypair.go @@ -0,0 +1,33 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + // todo: 需要进一步确认 + type KeyPairListOptions struct { + } + shellutils.R(&KeyPairListOptions{}, "keypair-list", "List keypairs", func(cli *huawei.SRegion, args *KeyPairListOptions) error { + keypairs, total, e := cli.GetKeypairs() + if e != nil { + return e + } + printList(keypairs, total, 0, 0, []string{}) + return nil + }) + + type KeyPairImportOptions struct { + NAME string `help:"Name of new keypair"` + PUBKEY string `help:"Public key string"` + } + shellutils.R(&KeyPairImportOptions{}, "keypair-import", "Import a keypair", func(cli *huawei.SRegion, args *KeyPairImportOptions) error { + keypair, err := cli.ImportKeypair(args.NAME, args.PUBKEY) + if err != nil { + return err + } + printObject(keypair) + return nil + }) +} diff --git a/pkg/util/huawei/shell/network.go b/pkg/util/huawei/shell/network.go new file mode 100644 index 0000000000..27cada1b1b --- /dev/null +++ b/pkg/util/huawei/shell/network.go @@ -0,0 +1,22 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type VSwitchListOptions struct { + Vpc string `help:"Vpc ID"` + Limit int `help:"page size"` + Marker string `help:"page marker"` + } + shellutils.R(&VSwitchListOptions{}, "vswitch-list", "List vswitches", func(cli *huawei.SRegion, args *VSwitchListOptions) error { + vswitches, total, e := cli.GetNetwroks(args.Vpc, args.Limit, args.Marker) + if e != nil { + return e + } + printList(vswitches, total, 0, args.Limit, []string{}) + return nil + }) +} diff --git a/pkg/util/huawei/shell/printutils.go b/pkg/util/huawei/shell/printutils.go new file mode 100644 index 0000000000..6d573bce88 --- /dev/null +++ b/pkg/util/huawei/shell/printutils.go @@ -0,0 +1,11 @@ +package shell + +import "yunion.io/x/onecloud/pkg/util/printutils" + +func printList(data interface{}, total, offset, limit int, columns []string) { + printutils.PrintInterfaceList(data, total, offset, limit, columns) +} + +func printObject(obj interface{}) { + printutils.PrintInterfaceObject(obj) +} diff --git a/pkg/util/huawei/shell/region.go b/pkg/util/huawei/shell/region.go new file mode 100644 index 0000000000..003a21aa81 --- /dev/null +++ b/pkg/util/huawei/shell/region.go @@ -0,0 +1,16 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type RegionListOptions struct { + } + shellutils.R(&RegionListOptions{}, "region-list", "List regions", func(cli *huawei.SRegion, args *RegionListOptions) error { + regions := cli.GetClient().GetRegions() + printList(regions, 0, 0, 0, nil) + return nil + }) +} diff --git a/pkg/util/huawei/shell/secgroup.go b/pkg/util/huawei/shell/secgroup.go new file mode 100644 index 0000000000..8adac0935f --- /dev/null +++ b/pkg/util/huawei/shell/secgroup.go @@ -0,0 +1,34 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type SecurityGroupListOptions struct { + VpcId string `help:"VPC ID"` + Limit int `help:"page size"` + Marker string `help:"page marker"` + } + shellutils.R(&SecurityGroupListOptions{}, "security-group-list", "List security group", func(cli *huawei.SRegion, args *SecurityGroupListOptions) error { + secgrps, total, e := cli.GetSecurityGroups(args.VpcId, args.Limit, args.Marker) + if e != nil { + return e + } + printList(secgrps, total, 0, 0, []string{}) + return nil + }) + + type SecurityGroupShowOptions struct { + ID string `help:"ID or name of security group"` + } + shellutils.R(&SecurityGroupShowOptions{}, "security-group-show", "Show details of a security group", func(cli *huawei.SRegion, args *SecurityGroupShowOptions) error { + secgrp, err := cli.GetSecurityGroupDetails(args.ID) + if err != nil { + return err + } + printObject(secgrp) + return nil + }) +} diff --git a/pkg/util/huawei/shell/snapshot.go b/pkg/util/huawei/shell/snapshot.go new file mode 100644 index 0000000000..c78e817f44 --- /dev/null +++ b/pkg/util/huawei/shell/snapshot.go @@ -0,0 +1,43 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type SnapshotListOptions struct { + DiskId string `help:"Disk ID"` + Name string `help:"Snapshot Name"` + Limit int `help:"page size"` + Offset int `help:"page offset"` + } + shellutils.R(&SnapshotListOptions{}, "snapshot-list", "List snapshot", func(cli *huawei.SRegion, args *SnapshotListOptions) error { + if snapshots, total, err := cli.GetSnapshots(args.DiskId, args.Name, args.Offset, args.Limit); err != nil { + return err + } else { + printList(snapshots, total, args.Offset, args.Limit, []string{}) + return nil + } + }) + + type SnapshotDeleteOptions struct { + ID string `help:"Snapshot ID"` + } + + shellutils.R(&SnapshotDeleteOptions{}, "snapshot-delete", "Delete snapshot", func(cli *huawei.SRegion, args *SnapshotDeleteOptions) error { + return cli.DeleteSnapshot(args.ID) + }) + + type SnapshotCreateOptions struct { + DiskId string `help:"Disk ID"` + Name string `help:"Snapeshot Name"` + Desc string `help:"Snapshot Desc"` + } + + shellutils.R(&SnapshotCreateOptions{}, "snapshot-create", "Create snapshot", func(cli *huawei.SRegion, args *SnapshotCreateOptions) error { + _, err := cli.CreateSnapshot(args.DiskId, args.Name, args.Desc) + return err + }) + +} diff --git a/pkg/util/huawei/shell/vpc.go b/pkg/util/huawei/shell/vpc.go new file mode 100644 index 0000000000..efff3346ff --- /dev/null +++ b/pkg/util/huawei/shell/vpc.go @@ -0,0 +1,21 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type VpcListOptions struct { + Limit int `help:"page size"` + Marker string `help:"page marker"` + } + shellutils.R(&VpcListOptions{}, "vpc-list", "List vpcs", func(cli *huawei.SRegion, args *VpcListOptions) error { + vpcs, total, e := cli.GetVpcs(args.Limit, args.Marker) + if e != nil { + return e + } + printList(vpcs, total, 0, args.Limit, []string{}) + return nil + }) +} diff --git a/pkg/util/huawei/shell/zone.go b/pkg/util/huawei/shell/zone.go new file mode 100644 index 0000000000..60b8a3d62d --- /dev/null +++ b/pkg/util/huawei/shell/zone.go @@ -0,0 +1,20 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/util/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ZoneListOptions struct { + } + shellutils.R(&ZoneListOptions{}, "zone-list", "List zones", func(cli *huawei.SRegion, args *ZoneListOptions) error { + zones, e := cli.GetIZones() + if e != nil { + return e + } + + printList(zones, 0, 0, 0, []string{}) + return nil + }) +} diff --git a/pkg/util/huawei/snapshot.go b/pkg/util/huawei/snapshot.go index 0f88f3d4bb..c2d9f8288d 100644 --- a/pkg/util/huawei/snapshot.go +++ b/pkg/util/huawei/snapshot.go @@ -120,6 +120,7 @@ func (self *SSnapshot) Delete() error { return self.region.DeleteSnapshot(self.GetId()) } +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0051408627.html func (self *SRegion) GetSnapshots(diskId string, snapshotName string, offset int, limit int) ([]SSnapshot, int, error) { params := make(map[string]string) params["limit"] = fmt.Sprintf("%d", limit) @@ -147,7 +148,24 @@ func (self *SRegion) GetSnapshotById(snapshotId string) (SSnapshot, error) { return snapshot, err } +// 不能删除以autobk_snapshot_为前缀的快照。 +// 当快照状态为available、error状态时，才可以删除。 func (self *SRegion) DeleteSnapshot(snapshotId string) error { - // todo: implement me - return nil + return DoDelete(self.ecsClient.Snapshots.Delete, snapshotId, nil, nil) +} + +// https://support.huaweicloud.com/api-evs/zh-cn_topic_0051408624.html +// 目前已设置force字段。云硬盘处于挂载状态时，能强制创建快照。 +func (self *SRegion) CreateSnapshot(diskId, name, desc string) (string, error) { + params := jsonutils.NewDict() + snapshotObj := jsonutils.NewDict() + snapshotObj.Add(jsonutils.NewString(name), "name") + snapshotObj.Add(jsonutils.NewString(desc), "description") + snapshotObj.Add(jsonutils.NewString(diskId), "volume_id") + snapshotObj.Add(jsonutils.JSONTrue, "force") + params.Add(snapshotObj, "snapshot") + + snapshot := SSnapshot{} + err := DoCreate(self.ecsClient.Snapshots.Create, params, &snapshot) + return snapshot.ID, err } diff --git a/pkg/util/huawei/storage.go b/pkg/util/huawei/storage.go index 2812d24bec..f3775b9676 100644 --- a/pkg/util/huawei/storage.go +++ b/pkg/util/huawei/storage.go @@ -90,7 +90,7 @@ func (self *SStorage) GetStorageType() string { } func (self *SStorage) GetMediumType() string { - if self.storageType == models.STORAGE_GP2_SSD || self.storageType == models.STORAGE_IO1_SSD { + if self.storageType == models.STORAGE_HUAWEI_SSD { return models.DISK_TYPE_SSD } else { return models.DISK_TYPE_ROTATE @@ -115,8 +115,18 @@ func (self *SStorage) GetManagerId() string { } func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { - // todo: implement me - return nil, nil + diskId, err := self.zone.region.CreateDisk(self.zone.GetId(), self.storageType, name, sizeGb, "", desc) + if err != nil { + log.Errorf("createDisk fail %s", err) + return nil, err + } + disk, err := self.zone.region.GetDisk(diskId) + if err != nil { + log.Errorf("getDisk fail %s", err) + return nil, err + } + disk.storage = self + return disk, nil } func (self *SStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { diff --git a/pkg/util/huawei/storagecache.go b/pkg/util/huawei/storagecache.go index a365483252..f170501996 100644 --- a/pkg/util/huawei/storagecache.go +++ b/pkg/util/huawei/storagecache.go @@ -2,11 +2,21 @@ package huawei import ( "context" + "strings" + "time" "fmt" + "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/huawei/obs" + "yunion.io/x/onecloud/pkg/util/qemuimg" ) type SStoragecache struct { @@ -15,6 +25,10 @@ type SStoragecache struct { iimages []cloudprovider.ICloudImage } +func GetBucketName(regionId string, imageId string) string { + return fmt.Sprintf("imgcache-%s-%s", strings.ToLower(regionId), imageId) +} + func (self *SStoragecache) fetchImages() error { limit := 100 marker := "" @@ -92,16 +106,138 @@ func (self *SStoragecache) GetManagerId() string { return self.region.client.providerId } +// 目前支持使用vhd、zvhd、vmdk、qcow2、raw、zvhd2、vhdx、qcow、vdi或qed格式镜像文件创建私有镜像。 +// 快速通道功能可快速完成镜像制作，但镜像文件需转换为raw或zvhd2格式并完成镜像优化。 func (self *SStoragecache) CreateIImage(snapshotId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { - panic("implement me") + if imageId, err := self.region.createIImage(snapshotId, imageName, imageDesc); err != nil { + return nil, err + } else if image, err := self.region.GetImage(imageId); err != nil { + return nil, err + } else { + image.storageCache = self + iimage := make([]cloudprovider.ICloudImage, 1) + iimage[0] = &image + if err := cloudprovider.WaitStatus(iimage[0], "avaliable", 15*time.Second, 3600*time.Second); err != nil { + return nil, err + } + return iimage[0], nil + } } func (self *SStoragecache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error) { - panic("implement me") + return self.downloadImage(userCred, imageId, extId) +} + +func (self *SStoragecache) downloadImage(userCred mcclient.TokenCredential, imageId string, extId string) (jsonutils.JSONObject, error) { + return nil, cloudprovider.ErrNotImplemented } func (self *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist, osVersion string, extId string, isForce bool) (string, error) { - panic("implement me") + if len(extId) > 0 { + log.Debugf("UploadImage: Image external ID exists %s", extId) + + image, err := self.region.GetImage(extId) + if err != nil { + log.Errorf("GetImageStatus error %s", err) + } + if image.Status == ImageStatusActive && !isForce { + return extId, nil + } + } else { + log.Debugf("UploadImage: no external ID") + } + + return self.uploadImage(ctx, userCred, imageId, osArch, osType, osDist, osVersion, isForce) +} + +func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, osVersion string, isForce bool) (string, error) { + bucketName := GetBucketName(self.region.GetId(), imageId) + obsClient, err := self.region.getOBSClient() + if err != nil { + return "", err + } + + // create bucket + input := &obs.CreateBucketInput{} + input.Bucket = bucketName + _, err = obsClient.CreateBucket(input) + if err != nil { + return "", err + } + defer obsClient.DeleteBucket(bucketName) + + // upload to huawei cloud + s := auth.GetAdminSession(ctx, options.Options.Region, "") + meta, reader, err := modules.Images.Download(s, imageId, string(qemuimg.VMDK), false) + if err != nil { + return "", err + } + log.Debugf("Images meta data %s", meta) + _image, err := modules.Images.Get(s, imageId, nil) + if err != nil { + return "", err + } + + minDiskGB, _ := _image.Int("min_disk") + if minDiskGB <= 0 { + minDiskGB = 40 + } + // upload to huawei cloud + obj := &obs.PutObjectInput{} + obj.Bucket = bucketName + obj.Key = imageId + obj.Body = reader + + _, err = obsClient.PutObject(obj) + if err != nil { + return "", err + } + + objDelete := &obs.DeleteObjectInput{} + objDelete.Bucket = bucketName + objDelete.Key = imageId + defer obsClient.DeleteObject(objDelete) // remove object + + // check image name, avoid name conflict + imageBaseName := imageId + if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { + imageBaseName = fmt.Sprintf("img%s", imageId) + } + imageName := imageBaseName + nameIdx := 1 + + for { + _, err = self.region.GetImageByName(imageName) + if err != nil { + if err == cloudprovider.ErrNotFound { + break + } else { + return "", err + } + } + + imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx) + nameIdx += 1 + log.Debugf("uploadImage Match remote name %s", imageName) + } + + jobId, err := self.region.ImportImageJob(imageName, osDist, osVersion, osArch, bucketName, imageId, minDiskGB) + + if err != nil { + log.Errorf("ImportImage error %s %s %s %s", jobId, imageId, bucketName, err) + return "", err + } + + // timeout: 1hour = 3600 seconds + serviceType := self.region.ecsClient.Images.ServiceType() + err = self.region.waitTaskStatus(serviceType, jobId, TASK_SUCCESS, 15*time.Second, 3600*time.Second) + if err != nil { + log.Errorf("waitTaskStatus %s", err) + return "", err + } + + // https://support.huaweicloud.com/api-ims/zh-cn_topic_0022473688.html + return self.region.GetTaskEntityID(serviceType, jobId, "image_id") } func (self *SRegion) getStoragecache() *SStoragecache { @@ -110,3 +246,65 @@ func (self *SRegion) getStoragecache() *SStoragecache { } return self.storageCache } + +type SJob struct { + Status string `json:"status"` + Entities map[string]string `json:"entities"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + BeginTime string `json:"begin_time"` + EndTime string `json:"end_time"` + ErrorCode string `json:"error_code"` + FailReason string `json:"fail_reason"` +} + +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0020092109.html +func (self *SRegion) createIImage(snapshotId, imageName, imageDesc string) (string, error) { + snapshot, err := self.GetSnapshotById(snapshotId) + if err != nil { + return "", err + } + + disk, err := self.GetDisk(snapshot.VolumeID) + if err != nil { + return "", err + } + + if disk.GetDiskType() != models.DISK_TYPE_SYS { + return "", fmt.Errorf("disk type err, expected disk type %s", models.DISK_TYPE_SYS) + } + + if len(disk.Attachments) == 0 { + return "", fmt.Errorf("disk is not attached.") + } + + imageObj := jsonutils.NewDict() + imageObj.Add(jsonutils.NewString(disk.Attachments[0].ServerID), "instance_id") + imageObj.Add(jsonutils.NewString(imageName), "name") + imageObj.Add(jsonutils.NewString(imageDesc), "description") + + ret, err := self.ecsClient.Images.PerformAction2("action", "", imageObj, "") + if err != nil { + return "", err + } + + job := SJob{} + jobId, err := ret.GetString("job_id") + querys := map[string]string{"service_type": self.ecsClient.Images.ServiceType()} + err = DoGet(self.ecsClient.Jobs.Get, jobId, querys, &job) + if err != nil { + return "", err + } + + if job.Status == "SUCCESS" { + imageId, exists := job.Entities["image_id"] + if exists { + return imageId, nil + } else { + return "", fmt.Errorf("image id not found in create image job %s", job.JobID) + } + } else { + return "", fmt.Errorf("create image failed, %s", job.FailReason) + } + +} diff --git a/pkg/util/huawei/task.go b/pkg/util/huawei/task.go new file mode 100644 index 0000000000..0d3128c7b9 --- /dev/null +++ b/pkg/util/huawei/task.go @@ -0,0 +1,80 @@ +package huawei + +import ( + "time" +) + +func (self *SRegion) waitTaskStatus(serviceType string, taskId string, targetStatus string, interval time.Duration, timeout time.Duration) error { + start := time.Now() + for time.Now().Sub(start) < timeout { + status, err := self.GetTaskStatus(serviceType, taskId) + if err != nil { + return err + } + if status == targetStatus { + break + } else { + time.Sleep(interval) + } + } + return nil +} + +func (self *SRegion) GetTaskStatus(serviceType string, taskId string) (string, error) { + querys := map[string]string{"service_type": serviceType} + task, err := self.ecsClient.Jobs.Get(taskId, querys) + if err != nil { + return "", err + } + + return task.GetString("status") +} + +// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0022225398.html +// 数据结构 entities -> []job +func (self *SRegion) GetAllSubTaskEntityIDs(serviceType string, taskId string, entityKeyName string) ([]string, error) { + err := self.waitTaskStatus(serviceType, taskId, TASK_SUCCESS, 10*time.Second, 600*time.Second) + if err != nil { + return nil, err + } + + querys := map[string]string{"service_type": serviceType} + ret, err := self.ecsClient.Jobs.Get(taskId, querys) + if err != nil { + return nil, err + } + + entities, err := ret.GetArray("entities", "sub_jobs") + if err != nil { + return nil, err + } + + ids := make([]string, 0) + for i := range entities { + entity := entities[i] + rid, err := entity.GetString("entities", entityKeyName) + if err != nil { + return nil, err + } + + ids = append(ids, rid) + } + + return ids, nil +} + +// 数据结构 entities -> job +func (self *SRegion) GetTaskEntityID(serviceType string, taskId string, entityKeyName string) (string, error) { + err := self.waitTaskStatus(serviceType, taskId, TASK_SUCCESS, 10*time.Second, 600*time.Second) + if err != nil { + return "", err + } + + querys := map[string]string{"service_type": serviceType} + ret, err := self.ecsClient.Jobs.Get(taskId, querys) + if err != nil { + return "", err + } + + return ret.GetString("entities", entityKeyName) +} diff --git a/pkg/util/huawei/utils.go b/pkg/util/huawei/utils.go index 0fd0635ca0..d7b0fde092 100644 --- a/pkg/util/huawei/utils.go +++ b/pkg/util/huawei/utils.go @@ -5,6 +5,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/huawei/client/manager" "yunion.io/x/onecloud/pkg/util/huawei/client/responses" ) @@ -12,7 +14,37 @@ import ( // 常用的方法 type listFunc func(querys map[string]string) (*responses.ListResult, error) type getFunc func(id string, querys map[string]string) (jsonutils.JSONObject, error) -type listInCtxFunc func(ctx *manager.ManagerContext, spec string, querys map[string]string) (*responses.ListResult, error) +type createFunc func(params jsonutils.JSONObject) (jsonutils.JSONObject, error) +type updateFunc func(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) +type updateFunc2 func(ctx manager.IManagerContext, id string, spec string, params jsonutils.JSONObject, responseKey string) (jsonutils.JSONObject, error) +type deleteFunc func(id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) +type listInCtxFunc func(ctx manager.IManagerContext, querys map[string]string) (*responses.ListResult, error) +type listInCtxWithSpecFunc func(ctx manager.IManagerContext, spec string, querys map[string]string, responseKey string) (*responses.ListResult, error) + +func unmarshalResult(resp jsonutils.JSONObject, respErr error, result interface{}) error { + if respErr != nil { + switch e := respErr.(type) { + case *httputils.JSONClientError: + if e.Code == 404 { + return cloudprovider.ErrNotFound + } + return e + default: + return e + } + } + + if result == nil { + return nil + } + + err := resp.Unmarshal(result) + if err != nil { + log.Errorf("unmarshal json error %s", err) + } + + return err +} func DoList(doList listFunc, querys map[string]string, result interface{}) error { ret, err := doList(querys) @@ -36,21 +68,11 @@ func DoGet(doGet getFunc, id string, querys map[string]string, result interface{ } ret, err := doGet(id, querys) - if err != nil { - return err - } - - err = ret.Unmarshal(result) - if err != nil { - log.Errorf("unmarshal json error %s", err) - return err - } - - return nil + return unmarshalResult(ret, err, result) } -func DoListInContext(listFunc listInCtxFunc, ctx *manager.ManagerContext, querys map[string]string, result interface{}) error { - ret, err := listFunc(ctx, "", querys) +func DoListInContext(listFunc listInCtxFunc, ctx manager.IManagerContext, querys map[string]string, result interface{}) error { + ret, err := listFunc(ctx, querys) if err != nil { return err } @@ -64,3 +86,27 @@ func DoListInContext(listFunc listInCtxFunc, ctx *manager.ManagerContext, querys return nil } + +func DoCreate(createFunc createFunc, params jsonutils.JSONObject, result interface{}) error { + ret, err := createFunc(params) + return unmarshalResult(ret, err, result) +} + +func DoUpdate(updateFunc updateFunc, id string, params jsonutils.JSONObject, result interface{}) error { + ret, err := updateFunc(id, params) + return unmarshalResult(ret, err, result) +} + +func DoUpdateWithSpec(updateFunc updateFunc2, id string, spec string, params jsonutils.JSONObject) error { + _, err := updateFunc(nil, id, spec, params, "") + return err +} + +func DoDelete(deleteFunc deleteFunc, id string, params jsonutils.JSONObject, result interface{}) error { + if len(id) == 0 { + return fmt.Errorf(" id should not be empty") + } + + ret, err := deleteFunc(id, params) + return unmarshalResult(ret, err, result) +} diff --git a/pkg/util/huawei/vpc.go b/pkg/util/huawei/vpc.go index c32cf0b4c4..e6b0cfb61d 100644 --- a/pkg/util/huawei/vpc.go +++ b/pkg/util/huawei/vpc.go @@ -1,6 +1,8 @@ package huawei import ( + "strconv" + "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -27,15 +29,15 @@ func (self *SVpc) addWire(wire *SWire) { self.iwires = append(self.iwires, wire) } -func (self *SVpc) getWireByZoneId(zoneId string) *SWire { +func (self *SVpc) getWireByRegionId(regionId string) *SWire { + if len(regionId) == 0 { + return nil + } + for i := 0; i < len(self.iwires); i++ { wire := self.iwires[i].(*SWire) - if zoneId == "" { - return wire - } - - if wire.zone.ZoneName == zoneId { + if wire.region.GetId() == regionId { return wire } } @@ -61,8 +63,13 @@ func (self *SVpc) fetchNetworks() error { marker = parts[count-1].ID } + if len(networks) == 0 { + self.iwires = append(self.iwires, &SWire{region: self.region, vpc: self}) + return nil + } + for i := 0; i < len(networks); i += 1 { - wire := self.getWireByZoneId(networks[i].AvailabilityZone) + wire := self.getWireByRegionId(self.region.GetId()) networks[i].wire = wire wire.addNetwork(&networks[i]) } @@ -90,6 +97,7 @@ func (self *SVpc) fetchSecurityGroups() error { } self.secgroups = make([]cloudprovider.ICloudSecurityGroup, len(secgroups)) + // 这里已经填充了vpc。 所以是不是不需要在GetSecurityGroups方法中填充vpc和region了？ for i := 0; i < len(secgroups); i++ { secgroups[i].vpc = self self.secgroups[i] = &secgroups[i] @@ -175,7 +183,8 @@ func (self *SVpc) GetManagerId() string { } func (self *SVpc) Delete() error { - panic("implement me") + // todo: 确定删除VPC的逻辑 + return self.region.DeleteVpc(self.GetId()) } func (self *SVpc) GetIWireById(wireId string) (cloudprovider.ICloudWire, error) { @@ -196,5 +205,30 @@ func (self *SVpc) GetIWireById(wireId string) (cloudprovider.ICloudWire, error) func (self *SRegion) getVpc(vpcId string) (*SVpc, error) { vpc := SVpc{} err := DoGet(self.ecsClient.Vpcs.Get, vpcId, nil, &vpc) + vpc.region = self return &vpc, err } + +func (self *SRegion) DeleteVpc(vpcId string) error { + return DoDelete(self.ecsClient.Vpcs.Delete, vpcId, nil, nil) +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090625.html +func (self *SRegion) GetVpcs(limit int, marker string) ([]SVpc, int, error) { + querys := map[string]string{"limit": "100"} + if limit > 0 { + querys["limit"] = strconv.Itoa(limit) + } + + if len(marker) > 0 { + querys["marker"] = marker + } + + vpcs := make([]SVpc, 0) + err := DoList(self.ecsClient.Vpcs.List, querys, &vpcs) + + for i := range vpcs { + vpcs[i].region = self + } + return vpcs, len(vpcs), err +} diff --git a/pkg/util/huawei/wire.go b/pkg/util/huawei/wire.go index 2ae654ff56..475e548609 100644 --- a/pkg/util/huawei/wire.go +++ b/pkg/util/huawei/wire.go @@ -2,20 +2,24 @@ package huawei import ( "fmt" + "net" + "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" ) +// 华为云的子网有点特殊。子网在整个region可用。 type SWire struct { - zone *SZone - vpc *SVpc + region *SRegion + vpc *SVpc inetworks []cloudprovider.ICloudNetwork } func (self *SWire) GetId() string { - return fmt.Sprintf("%s-%s", self.vpc.GetId(), self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.vpc.GetId(), self.region.GetId()) } func (self *SWire) GetName() string { @@ -23,7 +27,7 @@ func (self *SWire) GetName() string { } func (self *SWire) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.vpc.GetGlobalId(), self.zone.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.vpc.GetGlobalId(), self.region.GetGlobalId()) } func (self *SWire) GetStatus() string { @@ -47,7 +51,7 @@ func (self *SWire) GetIVpc() cloudprovider.ICloudVpc { } func (self *SWire) GetIZone() cloudprovider.ICloudZone { - return self.zone + return nil } func (self *SWire) GetINetworks() ([]cloudprovider.ICloudNetwork, error) { @@ -77,8 +81,35 @@ func (self *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, e return nil, cloudprovider.ErrNotFound } +/* +华为云子网可用区，类似一个zone标签。即使指定了zone子网在整个region依然是可用。 +通过华为web控制台创建子网需要指定可用区。这里是不指定的。 +*/ func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - panic("implement me") + networkId, err := self.region.createNetwork(self.vpc.GetId(), name, cidr, desc) + if err != nil { + log.Errorf("createNetwork error %s", err) + return nil, err + } + + var network *SNetwork + err = cloudprovider.WaitCreated(5*time.Second, 60*time.Second, func() bool { + self.inetworks = nil + network = self.getNetworkById(networkId) + if network == nil { + return false + } else { + return true + } + }) + + if err != nil { + log.Errorf("cannot find network after create????") + return nil, err + } + + network.wire = self + return network, nil } func (self *SWire) addNetwork(network *SNetwork) { @@ -112,3 +143,43 @@ func (self *SWire) getNetworkById(networkId string) *SNetwork { } return nil } + +func getDefaultGateWay(cidr string) (string, error) { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return "", err + } + + ipv4 := ip.To4() + if ipv4 == nil || len(ip.String()) == net.IPv6len { + return "", fmt.Errorf("ipv6 is not supported currently") + } + + if ipv4[3] != 0 { + return "", fmt.Errorf("the last byte of ip address must be zero. e.g 192.168.0.0/16") + } + + ipv4[3] = 1 + return ipv4.String(), nil +} + +// https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090590.html +// cidr 掩码长度不能大于28 +func (self *SRegion) createNetwork(vpcId string, name string, cidr string, desc string) (string, error) { + gateway, err := getDefaultGateWay(cidr) + if err != nil { + return "", err + } + + params := jsonutils.NewDict() + subnetObj := jsonutils.NewDict() + subnetObj.Add(jsonutils.NewString(name), "name") + subnetObj.Add(jsonutils.NewString(vpcId), "vpc_id") + subnetObj.Add(jsonutils.NewString(cidr), "cidr") + subnetObj.Add(jsonutils.NewString(gateway), "gateway_ip") + params.Add(subnetObj, "subnet") + + subnet := SNetwork{} + err = DoCreate(self.ecsClient.Subnets.Create, params, &subnet) + return subnet.ID, err +} diff --git a/pkg/util/huawei/zone.go b/pkg/util/huawei/zone.go index 007450b20e..3c42f550c0 100644 --- a/pkg/util/huawei/zone.go +++ b/pkg/util/huawei/zone.go @@ -158,3 +158,14 @@ func (self *SRegion) getZoneById(id string) (*SZone, error) { } return nil, fmt.Errorf("no such zone %s", id) } + +func (self *SZone) getNetworkById(networkId string) *SNetwork { + for i := 0; i < len(self.iwires); i += 1 { + wire := self.iwires[i].(*SWire) + net := wire.getNetworkById(networkId) + if net != nil { + return net + } + } + return nil +} diff --git a/pkg/util/imagetools/imagetools.go b/pkg/util/imagetools/imagetools.go index 3d7566337c..a8393204e6 100644 --- a/pkg/util/imagetools/imagetools.go +++ b/pkg/util/imagetools/imagetools.go @@ -97,11 +97,12 @@ func normalizeOsVersion(imageName string, osDist string, osVersion string) strin } type ImageInfo struct { - Name string - OsArch string - OsType string - OsDistro string - OsVersion string + Name string + OsArch string + OsType string + OsDistro string + OsVersion string + OsFullVersion string } func NormalizeImageInfo(imageName, osArch, osType, osDist, osVersion string) ImageInfo { @@ -111,5 +112,6 @@ func NormalizeImageInfo(imageName, osArch, osType, osDist, osVersion string) Ima info.OsType = normalizeOsType(osType, info.OsDistro) info.OsArch = normalizeOsArch(osArch, info.OsType, info.OsDistro) info.OsVersion = normalizeOsVersion(imageName, info.OsDistro, osVersion) + info.OsFullVersion = osVersion return info } diff --git a/pkg/util/openstack/region.go b/pkg/util/openstack/region.go index 844bc24e73..b7f11281d5 100644 --- a/pkg/util/openstack/region.go +++ b/pkg/util/openstack/region.go @@ -268,7 +268,7 @@ func (region *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) { return nil, cloudprovider.ErrNotImplemented } -func (region *SRegion) CreateEIP(name string, bwMbps int, chargeType string) (cloudprovider.ICloudEIP, error) { +func (region *SRegion) CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (cloudprovider.ICloudEIP, error) { return nil, cloudprovider.ErrNotSupported } diff --git a/pkg/util/qcloud/eip.go b/pkg/util/qcloud/eip.go index 8334eaaaa2..7293ec9581 100644 --- a/pkg/util/qcloud/eip.go +++ b/pkg/util/qcloud/eip.go @@ -266,7 +266,7 @@ func (region *SRegion) AllocateEIP(name string, bwMbps int, chargeType TInternet return nil, cloudprovider.ErrNotFound } -func (region *SRegion) CreateEIP(name string, bwMbps int, chargeType string) (cloudprovider.ICloudEIP, error) { +func (region *SRegion) CreateEIP(name string, bwMbps int, chargeType string, bgpType string) (cloudprovider.ICloudEIP, error) { var ctype TInternetChargeType switch chargeType { case models.EIP_CHARGE_TYPE_BY_TRAFFIC: From 4dc2e05a6e579c1d61b1824d4614bad6591b640b Mon Sep 17 00:00:00 2001 From: TangBin Date: Sat, 19 Jan 2019 19:04:05 +0800 Subject: [PATCH 2/5] huaweicli support unsubscirbe instance --- pkg/util/huawei/client/modules/mod_orders.go | 7 +++++++ pkg/util/huawei/instance.go | 17 ++++++++++++++++- pkg/util/huawei/shell/instance.go | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/util/huawei/client/modules/mod_orders.go b/pkg/util/huawei/client/modules/mod_orders.go index f542321634..6df268518f 100644 --- a/pkg/util/huawei/client/modules/mod_orders.go +++ b/pkg/util/huawei/client/modules/mod_orders.go @@ -68,3 +68,10 @@ func (self *SOrderManager) Get(id string, querys map[string]string) (jsonutils.J // ！！！特殊调用 return self.GetInContextWithSpec(self.orderCtx, "orders-resource", id, querys, "") } + +func (self *SOrderManager) PerformAction(action string, id string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { + request := self.newRequest("POST", id, action, self.orderCtx) + request.SetContent([]byte(getContent(params))) + + return self._do(request, "") +} diff --git a/pkg/util/huawei/instance.go b/pkg/util/huawei/instance.go index c2e471c013..b590bb5cf3 100644 --- a/pkg/util/huawei/instance.go +++ b/pkg/util/huawei/instance.go @@ -658,11 +658,12 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st return "", err } - // 按需计费 var ids []string if params.Extendparam.ChargingMode == POST_PAID { + // 按需计费 ids, err = self.GetAllSubTaskEntityIDs(self.ecsClient.Servers.ServiceType(), _id, "server_id") } else { + // 包年包月 err = cloudprovider.WaitCreated(10*time.Second, 180*time.Second, func() bool { ids, err = self.getAllResIdsByType(_id, RESOURCE_TYPE_VM) if err != nil { @@ -997,3 +998,17 @@ func (self *SRegion) GetInstanceSecrityGroupIds(instanceId string) ([]string, er return securitygroupIds, nil } + +// https://support.huaweicloud.com/api-oce/zh-cn_topic_0082522030.html +func (self *SRegion) UnsubscribeInstance(instanceId string, domianId string) (jsonutils.JSONObject, error) { + unsubObj := jsonutils.NewDict() + unsubObj.Add(jsonutils.NewInt(1), "unSubType") + unsubObj.Add(jsonutils.NewInt(5), "unsubscribeReasonType") + unsubObj.Add(jsonutils.NewString("no reason"), "unsubscribeReason") + resList := jsonutils.NewArray() + resList.Add(jsonutils.NewString(instanceId)) + unsubObj.Add(resList, "resourceIds") + + self.ecsClient.Orders.SetDomainId(domianId) + return self.ecsClient.Orders.PerformAction("resources/delete", "", unsubObj) +} diff --git a/pkg/util/huawei/shell/instance.go b/pkg/util/huawei/shell/instance.go index ec48ce9401..9834c36891 100644 --- a/pkg/util/huawei/shell/instance.go +++ b/pkg/util/huawei/shell/instance.go @@ -137,4 +137,22 @@ func init() { } return nil }) + + type InstanceOrderUnsubscribeOptions struct { + ID string `help:"instance ID"` + DOMAIN string `help:"domain ID"` + } + + shellutils.R(&InstanceOrderUnsubscribeOptions{}, "instance-order-unsubscribe", "Unsubscribe a prepaid server", func(cli *huawei.SRegion, args *InstanceOrderUnsubscribeOptions) error { + instance, e := cli.GetInstanceByID(args.ID) + if e != nil { + return e + } + + _, err := cli.UnsubscribeInstance(instance.GetId(), args.DOMAIN) + if err != nil { + return err + } + return nil + }) } From e1f54c7e14e15d641e23ac3d6559ea2549e211d6 Mon Sep 17 00:00:00 2001 From: TangBin Date: Mon, 21 Jan 2019 11:23:23 +0800 Subject: [PATCH 3/5] add more comment --- pkg/util/huawei/image.go | 13 +++++++++---- pkg/util/huawei/instance.go | 1 + pkg/util/huawei/storagecache.go | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/util/huawei/image.go b/pkg/util/huawei/image.go index f040a43bbe..ace5a05efd 100644 --- a/pkg/util/huawei/image.go +++ b/pkg/util/huawei/image.go @@ -172,10 +172,15 @@ func (self *SRegion) GetImageByName(name string) (*SImage, error) { return &images[0], nil } -// https://support.huaweicloud.com/api-ims/zh-cn_topic_0020092109.html -// os version 取值范围： https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html -// todo: 考虑使用镜像快速导入。 https://support.huaweicloud.com/api-ims/zh-cn_topic_0133188204.html -// 使用OBS文件创建镜像 +/* https://support.huaweicloud.com/api-ims/zh-cn_topic_0020092109.html + os version 取值范围： https://support.huaweicloud.com/api-ims/zh-cn_topic_0031617666.html + 用于创建私有镜像的源云服务器系统盘大小大于等于40GB且不超过1024GB。 + 目前支持vhd，zvhd、raw，qcow2 + todo: 考虑使用镜像快速导入。 https://support.huaweicloud.com/api-ims/zh-cn_topic_0133188204.html + 使用OBS文件创建镜像 + + * openstack原生接口支持的格式：https://support.huaweicloud.com/api-ims/zh-cn_topic_0031615566.html +*/ func (self *SRegion) ImportImageJob(name string, osDist string, osVersion string, osArch string, bucket string, key string, minDiskGB int64) (string, error) { os_version, err := stdVersion(osDist, osVersion, osArch) log.Debugf("%s %s %s: %s", osDist, osVersion, osArch, os_version) diff --git a/pkg/util/huawei/instance.go b/pkg/util/huawei/instance.go index b590bb5cf3..dcf5c2cf32 100644 --- a/pkg/util/huawei/instance.go +++ b/pkg/util/huawei/instance.go @@ -867,6 +867,7 @@ func (self *SRegion) ChangeRoot(ctx context.Context, instanceId, imageId, passwd // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212692.html // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0110109377.html +// 一键式重置密码 需要安装安装一键式重置密码插件 https://support.huaweicloud.com/usermanual-ecs/zh-cn_topic_0068095385.html // 目前不支持直接重置密钥 func (self *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, deleteKeypair bool, description string) error { serverObj := jsonutils.NewDict() diff --git a/pkg/util/huawei/storagecache.go b/pkg/util/huawei/storagecache.go index f170501996..9c523311b5 100644 --- a/pkg/util/huawei/storagecache.go +++ b/pkg/util/huawei/storagecache.go @@ -108,6 +108,7 @@ func (self *SStoragecache) GetManagerId() string { // 目前支持使用vhd、zvhd、vmdk、qcow2、raw、zvhd2、vhdx、qcow、vdi或qed格式镜像文件创建私有镜像。 // 快速通道功能可快速完成镜像制作，但镜像文件需转换为raw或zvhd2格式并完成镜像优化。 +// https://support.huaweicloud.com/api-ims/zh-cn_topic_0083905788.html func (self *SStoragecache) CreateIImage(snapshotId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { if imageId, err := self.region.createIImage(snapshotId, imageName, imageDesc); err != nil { return nil, err From 8ff0c8cf38ce2a2ae220efa1f866cb94ef0fd4fd Mon Sep 17 00:00:00 2001 From: TangBin Date: Mon, 21 Jan 2019 14:55:45 +0800 Subject: [PATCH 4/5] add instance price key --- pkg/util/huawei/instance.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/util/huawei/instance.go b/pkg/util/huawei/instance.go index dcf5c2cf32..3f51f78a20 100644 --- a/pkg/util/huawei/instance.go +++ b/pkg/util/huawei/instance.go @@ -202,7 +202,13 @@ func (self *SInstance) GetInstanceType() string { func (self *SInstance) GetMetadata() *jsonutils.JSONDict { data := jsonutils.NewDict() - // todo: add price_key here + // cn-north-1::et2.2xlarge.16::win + lowerOs := self.GetOSType() + if strings.HasPrefix(lowerOs, "win") { + lowerOs = "win" + } + priceKey := fmt.Sprintf("%s::%s::%s", self.host.zone.region.GetId(), self.GetInstanceType(), lowerOs) + data.Add(jsonutils.NewString(priceKey), "price_key") data.Add(jsonutils.NewString(self.host.zone.GetGlobalId()), "zone_ext_id") if len(self.Image.ID) > 0 { if image, err := self.host.zone.region.GetImage(self.Image.ID); err != nil { From 9007c966cf69947f95343ea1b1c33d1d5ff32df3 Mon Sep 17 00:00:00 2001 From: TangBin Date: Thu, 24 Jan 2019 11:56:25 +0800 Subject: [PATCH 5/5] fix comment --- pkg/compute/tasks/eip_allocate_task.go | 2 +- pkg/util/huawei/securitygroup.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/compute/tasks/eip_allocate_task.go b/pkg/compute/tasks/eip_allocate_task.go index 22bcf0a43d..c3eaaaeb2d 100644 --- a/pkg/compute/tasks/eip_allocate_task.go +++ b/pkg/compute/tasks/eip_allocate_task.go @@ -45,7 +45,7 @@ func (self *EipAllocateTask) OnInit(ctx context.Context, obj db.IStandaloneModel return } - extEip, err := iregion.CreateEIP(eip.Name, eip.Bandwidth, eip.ChargeType, "") + extEip, err := iregion.CreateEIP(eip.Name, eip.Bandwidth, eip.ChargeType, eip.BgpType) if err != nil { msg := fmt.Sprintf("create eip fail %s", err) eip.SetStatus(self.UserCred, models.EIP_STATUS_ALLOCATE_FAIL, msg) diff --git a/pkg/util/huawei/securitygroup.go b/pkg/util/huawei/securitygroup.go index 5638e7df02..ce685575db 100644 --- a/pkg/util/huawei/securitygroup.go +++ b/pkg/util/huawei/securitygroup.go @@ -183,6 +183,10 @@ func (self *SSecurityGroup) GetSecurityRule(ruleId string, withRuleId bool) (sec _, ipNet, err = net.ParseCIDR("0.0.0.0/0") } + if err != nil { + return secrules.SecurityRule{}, err + } + // withRuleId.将ruleId附加到description字段。该hook有特殊目的，仅在同步安全组时使用。 desc := "" if withRuleId { @@ -202,6 +206,8 @@ func (self *SSecurityGroup) GetSecurityRule(ruleId string, withRuleId bool) (sec Ports: nil, Description: desc, } + + err = rule.ValidateRule() return rule, err }