diff --git a/cmd/climc/shell/cloudaccounts.go b/cmd/climc/shell/cloudaccounts.go index fb378a1902..ef1b7f3a4f 100644 --- a/cmd/climc/shell/cloudaccounts.go +++ b/cmd/climc/shell/cloudaccounts.go @@ -661,6 +661,23 @@ func init() { return nil }) + R(&options.SGoogleCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-google", "Update credential of a Google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountUpdateCredentialOptions) error { + data, err := ioutil.ReadFile(args.GoogleJsonFile) + if err != nil { + return err + } + params, err := jsonutils.Parse(data) + if err != nil { + return err + } + result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params) + if err != nil { + return err + } + printObject(result) + return nil + }) + R(&options.SS3CloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-s3", "Update credential of a generic S3 cloud account", func(s *mcclient.ClientSession, args *options.SS3CloudAccountUpdateCredentialOptions) error { params := jsonutils.Marshal(args) result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params) diff --git a/cmd/climc/shell/hosts.go b/cmd/climc/shell/hosts.go index 910fe8637d..8a000a1e05 100644 --- a/cmd/climc/shell/hosts.go +++ b/cmd/climc/shell/hosts.go @@ -38,7 +38,7 @@ func init() { Occupied bool `help:"show occupid host" json:"-"` Enabled bool `help:"Show enabled host only" json:"-"` Disabled bool `help:"Show disabled host only" json:"-"` - HostType string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud"` + HostType string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv|aliyun|azure|qcloud|aws|huawei|ucloud|google"` AnyMac string `help:"Mac matches one of the host's interface"` IsBaremetal *bool `help:"filter host list by is_baremetal=true|false"` diff --git a/cmd/climc/shell/usages.go b/cmd/climc/shell/usages.go index 35f2cfd621..4fb79261a7 100644 --- a/cmd/climc/shell/usages.go +++ b/cmd/climc/shell/usages.go @@ -22,9 +22,9 @@ import ( ) type GeneralUsageOptions struct { - HostType []string `help:"Host types" choices:"hypervisor|baremetal|esxi|xen|kubelet|hyperv|aliyun|azure|aws|huawei|qcloud|openstack|ucloud|zstack"` - Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack"` - Brand []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|DStack"` + HostType []string `help:"Host types" choices:"hypervisor|baremetal|esxi|xen|kubelet|hyperv|aliyun|azure|aws|huawei|qcloud|openstack|ucloud|zstack|google"` + Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|Google"` + Brand []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|DStack|Google"` Project string `help:"show usage of specified project"` ProjectDomain string `help:"show usage of specified domain"` diff --git a/pkg/apis/compute/host_const.go b/pkg/apis/compute/host_const.go index 4dee7e482e..b4ca9513f7 100644 --- a/pkg/apis/compute/host_const.go +++ b/pkg/apis/compute/host_const.go @@ -110,6 +110,7 @@ var HOST_TYPES = []string{ HOST_TYPE_UCLOUD, HOST_TYPE_ZSTACK, HOST_TYPE_CTYUN, + HOST_TYPE_GOOGLE, } var NIC_TYPES = []string{NIC_TYPE_IPMI, NIC_TYPE_ADMIN} diff --git a/pkg/cloudprovider/instance.go b/pkg/cloudprovider/instance.go index ac1634010f..7f24c7136f 100644 --- a/pkg/cloudprovider/instance.go +++ b/pkg/cloudprovider/instance.go @@ -15,8 +15,6 @@ package cloudprovider import ( - "fmt" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/osprofile" @@ -34,8 +32,9 @@ type SDiskInfo struct { } const ( - CLOUD_SHELL = "cloud-shell" - CLOUD_CONFIG = "cloud-config" + CLOUD_SHELL = "cloud-shell" + CLOUD_SHELL_WITHOUT_ENCRYPT = "cloud-shell-without-encrypt" + CLOUD_CONFIG = "cloud-config" ) type SManagedVMCreateConfig struct { @@ -69,6 +68,15 @@ type SManagedVMChangeConfig struct { InstanceType string } +type SManagedVMRebuildRootConfig struct { + Account string + Password string + ImageId string + PublicKey string + SysSizeGB int + OsType string +} + func (vmConfig *SManagedVMCreateConfig) GetConfig(config *jsonutils.JSONDict) error { if err := config.Unmarshal(vmConfig, "desc"); err != nil { return err @@ -120,9 +128,6 @@ func generateUserData(adminPublicKey, projectPublicKey, oUserData string) string } func (vmConfig *SManagedVMCreateConfig) InjectPasswordByCloudInit() error { - if vmConfig.OsType != osprofile.OS_TYPE_LINUX { - return fmt.Errorf("Only support inject Linux password, current osType is %s", vmConfig.OsType) - } loginUser := cloudinit.NewUser(vmConfig.Account) loginUser.SudoPolicy(cloudinit.USER_SUDO_NOPASSWD) if len(vmConfig.PublicKey) > 0 { diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index a9f176e79e..07cfbfeee8 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -258,6 +258,8 @@ type ICloudVM interface { // GetStatus() string // GetRemoteStatus() string + GetSerialOutput(port int) (string, error) // 目前仅谷歌云windows机器会使用到此接口 + GetVcpuCount() int GetVmemSizeMB() int //MB GetBootOrder() string @@ -285,7 +287,7 @@ type ICloudVM interface { UpdateUserData(userData string) error - RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) + RebuildRoot(ctx context.Context, config *SManagedVMRebuildRootConfig) (string, error) DeployVM(ctx context.Context, name string, username string, password string, publicKey string, deleteKeypair bool, description string) error diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 06a5e16364..f4a755c38b 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -274,6 +274,10 @@ func (self *SBaseGuestDriver) RemoteDeployGuestForCreate(ctx context.Context, us return nil, cloudprovider.ErrNotSupported } +func (self *SBaseGuestDriver) RemoteActionAfterGuestCreated(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, ivm cloudprovider.ICloudVM, desc *cloudprovider.SManagedVMCreateConfig) { + return +} + func (self *SBaseGuestDriver) RemoteDeployGuestForDeploy(ctx context.Context, guest *models.SGuest, ihost cloudprovider.ICloudHost, task taskman.ITask, desc cloudprovider.SManagedVMCreateConfig) (jsonutils.JSONObject, error) { return nil, cloudprovider.ErrNotSupported } diff --git a/pkg/compute/guestdrivers/google.go b/pkg/compute/guestdrivers/google.go index c5df7cd972..fd299255cd 100644 --- a/pkg/compute/guestdrivers/google.go +++ b/pkg/compute/guestdrivers/google.go @@ -15,10 +15,26 @@ package guestdrivers import ( + "context" + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/multicloud/google" + "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/rbacutils" ) @@ -56,3 +72,189 @@ func (self *SGoogleGuestDriver) GetDefaultSysDiskBackend() string { func (self *SGoogleGuestDriver) GetMinimalSysDiskSizeGb() int { return 10 } + +func (self *SGoogleGuestDriver) GetStorageTypes() []string { + return []string{ + api.STORAGE_GOOGLE_PD_SSD, + api.STORAGE_GOOGLE_PD_STANDARD, + api.STORAGE_GOOGLE_LOCAL_SSD, + } +} + +func (self *SGoogleGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage { + return self.chooseHostStorage(self, host, backend, storageIds) +} + +func (self *SGoogleGuestDriver) GetGuestInitialStateAfterCreate() string { + return api.VM_RUNNING +} + +func (self *SGoogleGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SGoogleGuestDriver) GetAttachDiskStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SGoogleGuestDriver) GetRebuildRootStatus() ([]string, error) { + return []string{api.VM_READY}, nil +} + +func (self *SGoogleGuestDriver) GetChangeConfigStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SGoogleGuestDriver) GetDeployStatus() ([]string, error) { + return []string{api.VM_READY}, nil +} + +func (self *SGoogleGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { + if !utils.IsInStringArray(guest.Status, []string{api.VM_READY, api.VM_RUNNING}) { + return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status) + } + if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD}) { + return fmt.Errorf("Cannot resize %s disk", storage.StorageType) + } + return nil +} + +func (self *SGoogleGuestDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.ServerCreateInput) (*api.ServerCreateInput, error) { + input, err := self.SManagedVirtualizedGuestDriver.ValidateCreateData(ctx, userCred, input) + if err != nil { + return nil, err + } + if len(input.Networks) > 2 { + return nil, httperrors.NewInputParameterError("cannot support more than 1 nic") + } + localDisk := 0 + for i, disk := range input.Disks { + minGB := -1 + maxGB := -1 + switch disk.Backend { + case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD: + minGB = 10 + maxGB = 65536 + case api.STORAGE_GOOGLE_LOCAL_SSD: + minGB = 375 + maxGB = 375 + localDisk++ + } + if i == 0 && disk.Backend == api.STORAGE_GOOGLE_LOCAL_SSD { + return nil, httperrors.NewInputParameterError("System disk does not support %s disk", disk.Backend) + } + if disk.SizeMb < minGB*1024 || disk.SizeMb > maxGB*1024 { + return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of %dGB ~ %dGB", disk.Backend, minGB, maxGB) + } + } + if localDisk > 8 { + return nil, httperrors.NewInputParameterError("%s disk cannot exceed 8", api.STORAGE_GOOGLE_LOCAL_SSD) + } + return input, nil +} + +func (self *SGoogleGuestDriver) GetGuestInitialStateAfterRebuild() string { + return api.VM_READY +} + +func (self *SGoogleGuestDriver) IsNeedInjectPasswordByCloudInit(desc *cloudprovider.SManagedVMCreateConfig) bool { + return true +} + +// 谷歌云的用户自定义脚本不支持base64加密 +func (self *SGoogleGuestDriver) GetUserDataType() string { + return cloudprovider.CLOUD_SHELL_WITHOUT_ENCRYPT +} + +func (self *SGoogleGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { + ihost, err := host.GetIHost() + if err != nil { + return nil, errors.Wrap(err, "host.GetIHost") + } + + ivm, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil { + return nil, errors.Wrap(err, "ihost.GetIVMById") + } + + result := jsonutils.NewDict() + if ivm.GetStatus() != api.VM_RUNNING { + err := ivm.StartVM(ctx) + if err != nil { + return nil, errors.Wrap(err, "ivm.StartVM") + } + vm := ivm.(*google.SInstance) + updateUserdata := false + for _, item := range vm.Metadata.Items { + if item.Key == google.METADATA_STARTUP_SCRIPT || item.Key == google.METADATA_STARTUP_SCRIPT_POWER_SHELL { + updateUserdata = true + break + } + } + if updateUserdata { + keyword := "Finished running startup scripts" + err = cloudprovider.Wait(time.Second*5, time.Minute*6, func() (bool, error) { + output, err := ivm.GetSerialOutput(1) + if err != nil { + return false, errors.Wrap(err, "iVM.GetSerialOutput") + } + log.Debugf("wait for google startup scripts finish") + if strings.Contains(output, keyword) { + log.Debugf(keyword) + return true, nil + } + return false, nil + }) + if err != nil { + log.Errorf("failed wait google cloud startup scripts finish err: %v", err) + } + log.Debugf("clean google instance %s(%s) startup-script", guest.Name, guest.Id) + err := ivm.UpdateUserData("") + if err != nil { + log.Errorf("failed to update google userdata") + } + } + task.ScheduleRun(result) + } else { + result.Add(jsonutils.NewBool(true), "is_running") + } + + return result, nil +} + +func (self *SGoogleGuestDriver) RemoteActionAfterGuestCreated(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, iVM cloudprovider.ICloudVM, desc *cloudprovider.SManagedVMCreateConfig) { + keywords := map[string]string{ + strings.ToLower(osprofile.OS_TYPE_WINDOWS): "Finished with sysprep specialize phase", + strings.ToLower(osprofile.OS_TYPE_LINUX): "Finished running startup scripts", + } + if keyword, ok := keywords[strings.ToLower(desc.OsType)]; ok { + err := cloudprovider.Wait(time.Second*5, time.Minute*6, func() (bool, error) { + output, err := iVM.GetSerialOutput(1) + if err != nil { + return false, errors.Wrap(err, "iVM.GetSerialOutput") + } + log.Debugf("wait for google sysprep finish") + if strings.Contains(output, keyword) { + log.Debugf(keyword) + return true, nil + } + return false, nil + }) + if err != nil { + log.Errorf("failed wait google %s error: %v", keyword, err) + } + } + log.Debugf("clean google instance %s(%s) startup-script", guest.Name, guest.Id) + err := iVM.UpdateUserData("") + if err != nil { + log.Errorf("failed to update google userdata") + } +} + +func (self *SGoogleGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SGoogleGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool { + return false +} diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 046a5d4a6a..809ab26d4d 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -18,11 +18,13 @@ import ( "context" "fmt" "math" + "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/utils" billing_api "yunion.io/x/onecloud/pkg/apis/billing" @@ -153,7 +155,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestDetachDisk(ctx context.Contex iVM, err := guest.GetIVM() if err != nil { //若guest被删除,忽略错误，否则会无限删除guest失败(有挂载的云盘) - if err == cloudprovider.ErrNotFound { + if errors.Cause(err) == cloudprovider.ErrNotFound { return nil, nil } return nil, errors.Wrapf(err, "guest.GetIVM") @@ -338,9 +340,14 @@ func (self *SManagedVirtualizedGuestDriver) RequestDeployGuestOnHost(ctx context switch guest.GetDriver().GetUserDataType() { case cloudprovider.CLOUD_SHELL: desc.UserData = oUserData.UserDataScriptBase64() + case cloudprovider.CLOUD_SHELL_WITHOUT_ENCRYPT: + desc.UserData = oUserData.UserDataScript() default: desc.UserData = oUserData.UserDataBase64() } + if strings.ToLower(desc.OsType) == strings.ToLower(osprofile.OS_TYPE_WINDOWS) { + desc.UserData = oUserData.UserDataPowerShell() + } } action, err := config.GetString("action") @@ -396,7 +403,11 @@ func (self *SManagedVirtualizedGuestDriver) GetGuestInitialStateAfterRebuild() s } func (self *SManagedVirtualizedGuestDriver) GetLinuxDefaultAccount(desc cloudprovider.SManagedVMCreateConfig) string { - return "root" + userName := "root" + if strings.ToLower(desc.OsType) == strings.ToLower(osprofile.OS_TYPE_WINDOWS) { + userName = "Administrator" + } + return userName } func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, host *models.SHost, desc cloudprovider.SManagedVMCreateConfig) (jsonutils.JSONObject, error) { @@ -467,6 +478,8 @@ func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForCreate(ctx conte return nil, err } + guest.GetDriver().RemoteActionAfterGuestCreated(ctx, userCred, guest, host, iVM, &desc) + data := fetchIVMinfo(desc, iVM, guest.Id, desc.Account, desc.Password, desc.PublicKey, "create") return data, nil } @@ -541,7 +554,15 @@ func (self *SManagedVirtualizedGuestDriver) RemoteDeployGuestForRebuildRoot(ctx lockman.LockObject(ctx, guest) defer lockman.ReleaseObject(ctx, guest) - return iVM.RebuildRoot(ctx, desc.ExternalImageId, desc.Password, desc.PublicKey, desc.SysDisk.SizeGB) + conf := cloudprovider.SManagedVMRebuildRootConfig{ + Account: desc.Account, + ImageId: desc.ExternalImageId, + Password: desc.Password, + PublicKey: desc.PublicKey, + SysSizeGB: desc.SysDisk.SizeGB, + OsType: desc.OsType, + } + return iVM.RebuildRoot(ctx, &conf) }() if err != nil { return nil, err @@ -610,7 +631,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte ivm, err := ihost.GetIVMById(guest.ExternalId) if err != nil { - if err == cloudprovider.ErrNotFound { + if errors.Cause(err) == cloudprovider.ErrNotFound { return nil, nil } log.Errorf("ihost.GetIVMById fail %s", err) @@ -626,7 +647,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte if disk := guestdisk.GetDisk(); disk != nil && disk.AutoDelete { idisk, err := disk.GetIDisk() if err != nil { - if err == cloudprovider.ErrNotFound { + if errors.Cause(err) == cloudprovider.ErrNotFound { continue } log.Errorf("disk.GetIDisk fail %s", err) diff --git a/pkg/compute/guestdrivers/openstack.go b/pkg/compute/guestdrivers/openstack.go index cf99d9f92f..c1f91ad85a 100644 --- a/pkg/compute/guestdrivers/openstack.go +++ b/pkg/compute/guestdrivers/openstack.go @@ -199,7 +199,15 @@ func (self *SOpenStackGuestDriver) RemoteDeployGuestForRebuildRoot(ctx context.C } storage := sysDisk.GetStorage() if storage.StorageType == api.STORAGE_OPENSTACK_NOVA { //不通过镜像创建磁盘的机器 - return iVM.RebuildRoot(ctx, desc.ExternalImageId, desc.Password, desc.PublicKey, desc.SysDisk.SizeGB) + conf := cloudprovider.SManagedVMRebuildRootConfig{ + Account: desc.Account, + ImageId: desc.ExternalImageId, + Password: desc.Password, + PublicKey: desc.PublicKey, + SysSizeGB: desc.SysDisk.SizeGB, + OsType: desc.OsType, + } + return iVM.RebuildRoot(ctx, &conf) } iDisks, err := iVM.GetIDisks() diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 7f4f46e106..0987746b91 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -72,7 +72,7 @@ func (self *SAliyunHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb func (self *SAliyunHostDriver) ValidateResetDisk(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, snapshot *models.SSnapshot, guests []models.SGuest, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { for _, guest := range guests { if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) { - return nil, httperrors.NewBadGatewayError("Aliyun reset disk required guest status is running or read") + return nil, httperrors.NewBadGatewayError("Aliyun reset disk required guest status is running or ready") } } return data, nil diff --git a/pkg/compute/hostdrivers/google.go b/pkg/compute/hostdrivers/google.go index cb7273a393..d85df35990 100644 --- a/pkg/compute/hostdrivers/google.go +++ b/pkg/compute/hostdrivers/google.go @@ -15,8 +15,16 @@ package hostdrivers import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SGoogleHostDriver struct { @@ -35,3 +43,27 @@ func (self *SGoogleHostDriver) GetHostType() string { func (self *SGoogleHostDriver) GetHypervisor() string { return api.HYPERVISOR_GOOGLE } + +func (self *SGoogleHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb int) error { + minGB := 10 + maxGB := -1 + switch storage.StorageType { + case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD: + maxGB = 65536 + default: + return fmt.Errorf("Not support resize %s disk", storage.StorageType) + } + if sizeGb < minGB || sizeGb > maxGB { + return fmt.Errorf("The %s disk size must be in the range of %dG ~ %dGB", storage.StorageType, minGB, maxGB) + } + return nil +} + +func (self *SGoogleHostDriver) ValidateResetDisk(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, snapshot *models.SSnapshot, guests []models.SGuest, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + for _, guest := range guests { + if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) { + return nil, httperrors.NewBadGatewayError("%s reset disk required guest status is running or ready", self.GetHostType()) + } + } + return data, nil +} diff --git a/pkg/compute/hostdrivers/managedvirtual.go b/pkg/compute/hostdrivers/managedvirtual.go index 2db894581b..877c0d5171 100644 --- a/pkg/compute/hostdrivers/managedvirtual.go +++ b/pkg/compute/hostdrivers/managedvirtual.go @@ -285,7 +285,7 @@ func (self *SManagedVirtualizationHostDriver) RequestDeallocateDiskOnHost(ctx co iDisk, err := iCloudStorage.GetIDiskById(disk.GetExternalId()) if err != nil { - if err == cloudprovider.ErrNotFound { + if errors.Cause(err) == cloudprovider.ErrNotFound { task.ScheduleRun(data) return nil } diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 4d3d2a9478..189b97013e 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -726,6 +726,12 @@ func (self *SCloudregion) ValidateUpdateCondition(ctx context.Context) error { return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateCondition(ctx) } +func (self *SCloudregion) SyncVpcs(ctx context.Context, userCred mcclient.TokenCredential, iregion cloudprovider.ICloudRegion, provider *SCloudprovider) error { + syncResults, syncRange := SSyncResultSet{}, &SSyncRange{} + syncRegionVPCs(ctx, userCred, syncResults, provider, self, iregion, syncRange) + return nil +} + func (self *SCloudregion) AllowGetDetailsCapability(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { return true } diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 66cc588f4e..ef667f8209 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -88,6 +88,7 @@ type IGuestDriver interface { RequestDeployGuestOnHost(ctx context.Context, guest *SGuest, host *SHost, task taskman.ITask) error RemoteDeployGuestForCreate(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, host *SHost, desc cloudprovider.SManagedVMCreateConfig) (jsonutils.JSONObject, error) + RemoteActionAfterGuestCreated(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, host *SHost, iVM cloudprovider.ICloudVM, desc *cloudprovider.SManagedVMCreateConfig) RemoteDeployGuestForDeploy(ctx context.Context, guest *SGuest, ihost cloudprovider.ICloudHost, task taskman.ITask, desc cloudprovider.SManagedVMCreateConfig) (jsonutils.JSONObject, error) RemoteDeployGuestForRebuildRoot(ctx context.Context, guest *SGuest, ihost cloudprovider.ICloudHost, task taskman.ITask, desc cloudprovider.SManagedVMCreateConfig) (jsonutils.JSONObject, error) GetGuestInitialStateAfterCreate() string diff --git a/pkg/compute/models/regiondrivers.go b/pkg/compute/models/regiondrivers.go index d4fc1e3198..cd8423d628 100644 --- a/pkg/compute/models/regiondrivers.go +++ b/pkg/compute/models/regiondrivers.go @@ -87,6 +87,8 @@ type IRegionDriver interface { ValidateCreateVpcData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) ValidateCreateEipData(ctx context.Context, userCred mcclient.TokenCredential, input *api.SElasticipCreateInput) error + RequestCreateVpc(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, task taskman.ITask) error + RequestDeleteVpc(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, task taskman.ITask) error // Region Driver Snapshot Policy Apis //ValidateCreateSnapshotPolicyData(context.Context, mcclient.TokenCredential, *compute.SSnapshotPolicyCreateInput, mcclient.IIdentityProvider, *jsonutils.JSONDict) error diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index 4bbfae06cd..83455d7005 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -132,8 +132,20 @@ func (self *SSecurityGroupCache) GetCustomizeColumns(ctx context.Context, userCr func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId string, regionId string, providerId string) (*SSecurityGroupCache, error) { secgroupCache := SSecurityGroupCache{} query := manager.Query() - cond := sqlchemy.AND(sqlchemy.Equals(query.Field("secgroup_id"), secgroupId), sqlchemy.Equals(query.Field("vpc_id"), vpcId), sqlchemy.Equals(query.Field("cloudregion_id"), regionId), sqlchemy.Equals(query.Field("manager_id"), providerId)) - query = query.Filter(cond) + conds := []sqlchemy.ICondition{ + sqlchemy.Equals(query.Field("secgroup_id"), secgroupId), + sqlchemy.Equals(query.Field("vpc_id"), vpcId), + sqlchemy.Equals(query.Field("manager_id"), providerId), + } + _region, err := CloudregionManager.FetchById(regionId) + if err != nil { + return nil, errors.Wrapf(err, "CloudregionManager.FetchById(%s)", regionId) + } + region := _region.(*SCloudregion) + if !region.GetDriver().IsSecurityGroupBelongGlobalVpc() { + conds = append(conds, sqlchemy.Equals(query.Field("cloudregion_id"), regionId)) + } + query = query.Filter(sqlchemy.AND(conds...)) count, err := query.CountWithError() if err != nil { diff --git a/pkg/compute/regiondrivers/base.go b/pkg/compute/regiondrivers/base.go index 3e135be1e7..2f572dfd5e 100644 --- a/pkg/compute/regiondrivers/base.go +++ b/pkg/compute/regiondrivers/base.go @@ -203,6 +203,14 @@ func (self *SBaseRegionDriver) RequestBingToNatgateway(ctx context.Context, task return fmt.Errorf("Not implement RequestBindIPToNatgateway") } +func (self *SBaseRegionDriver) RequestCreateVpc(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, task taskman.ITask) error { + return fmt.Errorf("Not implement RequestCreateVpc") +} + +func (self *SBaseRegionDriver) RequestDeleteVpc(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, task taskman.ITask) error { + return fmt.Errorf("Not implement RequestDeleteVpc") +} + func (self *SBaseRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, task taskman.ITask) error { return fmt.Errorf("Not Implemented RequestCacheSecurityGroup") } diff --git a/pkg/compute/regiondrivers/google.go b/pkg/compute/regiondrivers/google.go index d020b8ff7a..f84b54b9b2 100644 --- a/pkg/compute/regiondrivers/google.go +++ b/pkg/compute/regiondrivers/google.go @@ -15,8 +15,18 @@ package regiondrivers import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" ) type SGoogleRegionDriver struct { @@ -39,3 +49,106 @@ func (self *SGoogleRegionDriver) IsSecurityGroupBelongGlobalVpc() bool { func (self *SGoogleRegionDriver) IsVpcBelongGlobalVpc() bool { return true } + +func (self *SGoogleRegionDriver) RequestCreateVpc(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + provider := vpc.GetCloudprovider() + if provider == nil { + return nil, fmt.Errorf("failed to found vpc %s(%s) cloudprovider", vpc.Name, vpc.Id) + } + providerDriver, err := provider.GetProvider() + if err != nil { + return nil, errors.Wrap(err, "provider.GetProvider") + } + iregion, err := providerDriver.GetIRegionById(region.ExternalId) + if err != nil { + return nil, errors.Wrap(err, "vpc.GetIRegion") + } + ivpc, err := iregion.CreateIVpc(vpc.Name, vpc.Description, vpc.CidrBlock) + if err != nil { + return nil, errors.Wrap(err, "iregion.CreateIVpc") + } + db.SetExternalId(vpc, userCred, ivpc.GetGlobalId()) + + regions, err := models.CloudregionManager.GetRegionByExternalIdPrefix(self.GetProvider()) + if err != nil { + return nil, errors.Wrap(err, "GetRegionByExternalIdPrefix") + } + for _, region := range regions { + iregion, err := providerDriver.GetIRegionById(region.ExternalId) + if err != nil { + return nil, errors.Wrap(err, "providerDrivder.GetIRegionById") + } + region.SyncVpcs(ctx, userCred, iregion, provider) + } + + err = vpc.SyncWithCloudVpc(ctx, userCred, ivpc) + if err != nil { + return nil, errors.Wrap(err, "vpc.SyncWithCloudVpc") + } + + err = vpc.SyncRemoteWires(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "vpc.SyncRemoteWires") + } + return nil, nil + }) + return nil +} + +func (self *SGoogleRegionDriver) RequestDeleteVpc(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + region, err := vpc.GetIRegion() + if err != nil { + return nil, errors.Wrap(err, "vpc.GetIRegion") + } + ivpc, err := region.GetIVpcById(vpc.GetExternalId()) + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + err = vpc.Purge(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "vpc.Purge") + } + return nil, nil + } + return nil, errors.Wrap(err, "region.GetIVpcById") + } + + globalVpc, err := vpc.GetGlobalVpc() + if err != nil { + return nil, errors.Wrap(err, "vpc.GetGlobalVpc") + } + + vpcs, err := globalVpc.GetVpcs() + if err != nil { + return nil, errors.Wrap(err, "globalVpc.GetVpcs") + } + + for i := range vpcs { + if vpcs[i].Status == api.VPC_STATUS_AVAILABLE && vpcs[i].ManagerId == vpc.ManagerId { + err = vpc.ValidateDeleteCondition(ctx) + if err != nil { + return nil, errors.Wrapf(err, "vpc %s(%s) not empty", vpc.Name, vpc.Id) + } + } + } + + err = ivpc.Delete() + if err != nil { + return nil, errors.Wrap(err, "ivpc.Delete") + } + + for i := range vpcs { + if vpcs[i].ManagerId == vpc.ManagerId && vpcs[i].Id != vpc.Id { + err = vpcs[i].Purge(ctx, userCred) + if err != nil { + return nil, errors.Wrapf(err, "vpc.Purge %s(%s)", vpc.Name, vpc.Id) + } + } + } + + return nil, nil + }) + return nil + +} diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index 4cdc4fb485..0315a0287a 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -1101,6 +1101,64 @@ func (self *SManagedVirtualizationRegionDriver) ValidateCreateEipData(ctx contex return nil } +func (self *SManagedVirtualizationRegionDriver) RequestCreateVpc(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iregion, err := vpc.GetIRegion() + if err != nil { + return nil, errors.Wrap(err, "vpc.GetIRegion") + } + ivpc, err := iregion.CreateIVpc(vpc.Name, vpc.Description, vpc.CidrBlock) + if err != nil { + return nil, errors.Wrap(err, "iregion.CreateIVpc") + } + db.SetExternalId(vpc, userCred, ivpc.GetGlobalId()) + + err = cloudprovider.WaitStatus(ivpc, api.VPC_STATUS_AVAILABLE, 10*time.Second, 300*time.Second) + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.WaitStatus") + } + + err = vpc.SyncWithCloudVpc(ctx, userCred, ivpc) + if err != nil { + return nil, errors.Wrap(err, "vpc.SyncWithCloudVpc") + } + + err = vpc.SyncRemoteWires(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "vpc.SyncRemoteWires") + } + return nil, nil + }) + return nil +} + +func (self *SManagedVirtualizationRegionDriver) RequestDeleteVpc(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, task taskman.ITask) error { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + region, err := vpc.GetIRegion() + if err != nil { + return nil, errors.Wrap(err, "vpc.GetIRegion") + } + ivpc, err := region.GetIVpcById(vpc.GetExternalId()) + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + // already deleted, do nothing + return nil, nil + } + return nil, errors.Wrap(err, "region.GetIVpcById") + } + err = ivpc.Delete() + if err != nil { + return nil, errors.Wrap(err, "ivpc.Delete(") + } + err = cloudprovider.WaitDeleted(ivpc, 10*time.Second, 300*time.Second) + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.WaitDeleted") + } + return nil, nil + }) + return nil +} + func (self *SManagedVirtualizationRegionDriver) RequestUpdateSnapshotPolicy(ctx context.Context, userCred mcclient. TokenCredential, sp *models.SSnapshotPolicy, input cloudprovider.SnapshotPolicyInput, task taskman.ITask) error { // it's too cumbersome to pass parameters in taskman, so change a simple way for the moment diff --git a/pkg/compute/tasks/security_group_cache_delete_task.go b/pkg/compute/tasks/security_group_cache_delete_task.go index e685c0fce3..bab8c0d7cf 100644 --- a/pkg/compute/tasks/security_group_cache_delete_task.go +++ b/pkg/compute/tasks/security_group_cache_delete_task.go @@ -67,7 +67,7 @@ func (self *SecurityGroupCacheDeleteTask) OnInit(ctx context.Context, obj db.ISt iRegion, err := cache.GetIRegion() if err != nil { - if err == cloudprovider.ErrNotFound { + if errors.Cause(err) == cloudprovider.ErrNotFound { self.taskComplete(ctx, cache) return } diff --git a/pkg/compute/tasks/vpc_create_task.go b/pkg/compute/tasks/vpc_create_task.go index 89a9393124..e0e786532c 100644 --- a/pkg/compute/tasks/vpc_create_task.go +++ b/pkg/compute/tasks/vpc_create_task.go @@ -16,15 +16,15 @@ package tasks import ( "context" - "time" + "fmt" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -49,36 +49,24 @@ func (self *VpcCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, vpc := obj.(*models.SVpc) vpc.SetStatus(self.UserCred, api.VPC_STATUS_PENDING, "") - iregion, err := vpc.GetIRegion() - if err != nil { - self.TaskFailed(ctx, vpc, err) - return - } - ivpc, err := iregion.CreateIVpc(vpc.Name, vpc.Description, vpc.CidrBlock) - if err != nil { - self.TaskFailed(ctx, vpc, err) - return - } - db.SetExternalId(vpc, self.UserCred, ivpc.GetGlobalId()) - - err = cloudprovider.WaitStatus(ivpc, api.VPC_STATUS_AVAILABLE, 10*time.Second, 300*time.Second) - if err != nil { - self.TaskFailed(ctx, vpc, err) - return - } - - err = vpc.SyncWithCloudVpc(ctx, self.UserCred, ivpc) - if err != nil { - self.TaskFailed(ctx, vpc, err) - return - } - - err = vpc.SyncRemoteWires(ctx, self.UserCred) + region, err := vpc.GetRegion() + if err != nil { + self.TaskFailed(ctx, vpc, errors.Wrap(err, "vpc.GetRegion")) + return + } + self.SetStage("OnCreateVpcComplete", nil) + err = region.GetDriver().RequestCreateVpc(ctx, self.UserCred, region, vpc, self) if err != nil { self.TaskFailed(ctx, vpc, err) return } +} +func (self *VpcCreateTask) OnCreateVpcComplete(ctx context.Context, vpc *models.SVpc, data jsonutils.JSONObject) { logclient.AddActionLogWithStartable(self, vpc, logclient.ACT_ALLOCATE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) } + +func (self *VpcCreateTask) OnCreateVpcCompleteFailed(ctx context.Context, vpc *models.SVpc, data jsonutils.JSONObject) { + self.TaskFailed(ctx, vpc, fmt.Errorf("%s", data.String())) +} diff --git a/pkg/compute/tasks/vpc_delete_task.go b/pkg/compute/tasks/vpc_delete_task.go index 2929c41efd..6cb2a2c000 100644 --- a/pkg/compute/tasks/vpc_delete_task.go +++ b/pkg/compute/tasks/vpc_delete_task.go @@ -16,15 +16,15 @@ package tasks import ( "context" - "time" + "fmt" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -47,40 +47,32 @@ func (self *VpcDeleteTask) taskFailed(ctx context.Context, vpc *models.SVpc, err func (self *VpcDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { vpc := obj.(*models.SVpc) - vpc.SetStatus(self.UserCred, api.VPC_STATUS_DELETING, "") + region, err := vpc.GetRegion() + if err != nil { + self.taskFailed(ctx, vpc, errors.Wrap(err, "vpc.GetRegion")) + return + } + + self.SetStage("OnDeleteVpcComplete", nil) + err = region.GetDriver().RequestDeleteVpc(ctx, self.UserCred, region, vpc, self) + if err != nil { + self.taskFailed(ctx, vpc, errors.Wrap(err, "RequestDeleteVpc")) + return + } +} + +func (self *VpcDeleteTask) OnDeleteVpcComplete(ctx context.Context, vpc *models.SVpc, body jsonutils.JSONObject) { + err := vpc.Purge(ctx, self.UserCred) + if err != nil { + self.taskFailed(ctx, vpc, errors.Wrap(err, "vpc.Purge")) + return + } db.OpsLog.LogEvent(vpc, db.ACT_DELOCATING, vpc.GetShortDesc(ctx), self.UserCred) - - region, err := vpc.GetIRegion() - if err != nil { - self.taskFailed(ctx, vpc, err) - return - } - ivpc, err := region.GetIVpcById(vpc.GetExternalId()) - if ivpc != nil { - err = ivpc.Delete() - if err != nil { - self.taskFailed(ctx, vpc, err) - return - } - err = cloudprovider.WaitDeleted(ivpc, 10*time.Second, 300*time.Second) - if err != nil { - self.taskFailed(ctx, vpc, err) - return - } - } else if err == cloudprovider.ErrNotFound { - // already deleted, do nothing - } else { - self.taskFailed(ctx, vpc, err) - return - } - - err = vpc.Purge(ctx, self.UserCred) - if err != nil { - self.taskFailed(ctx, vpc, err) - return - } - logclient.AddActionLogWithStartable(self, vpc, logclient.ACT_DELETE, nil, self.UserCred, true) self.SetStageComplete(ctx, nil) } + +func (self *VpcDeleteTask) OnDeleteVpcCompleteFailed(ctx context.Context, vpc *models.SVpc, reason jsonutils.JSONObject) { + self.taskFailed(ctx, vpc, fmt.Errorf("%s", reason)) +} diff --git a/pkg/mcclient/options/cloudaccounts.go b/pkg/mcclient/options/cloudaccounts.go index bb7af5a9cc..487740afdf 100644 --- a/pkg/mcclient/options/cloudaccounts.go +++ b/pkg/mcclient/options/cloudaccounts.go @@ -215,6 +215,11 @@ type SCtyunCloudAccountUpdateCredentialOptions struct { SAccessKeyCredential } +type SGoogleCloudAccountUpdateCredentialOptions struct { + SCloudAccountUpdateCredentialBaseOptions + GoogleJsonFile string `help:"Google auth json file" positional:"true"` +} + // update type SCloudAccountUpdateBaseOptions struct { diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 94d468af83..2fd8e4ce7d 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -129,7 +129,7 @@ type ServerConfigs struct { Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` BackupHost string `help:"Perfered host where virtual backup server should be created"` - Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei|openstack|ucloud|zstack"` + Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei|openstack|ucloud|zstack|google"` ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` Backup bool `help:"Create server with backup server"` diff --git a/pkg/multicloud/aliyun/instance.go b/pkg/multicloud/aliyun/instance.go index 92ed6c5d71..cb9c6d5d0e 100644 --- a/pkg/multicloud/aliyun/instance.go +++ b/pkg/multicloud/aliyun/instance.go @@ -493,16 +493,16 @@ func (self *SInstance) DeployVM(ctx context.Context, name string, username strin return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, deleteKeypair, description) } -func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { keypair := "" - if len(publicKey) > 0 { + if len(desc.PublicKey) > 0 { var err error - keypair, err = self.host.zone.region.syncKeypair(publicKey) + keypair, err = self.host.zone.region.syncKeypair(desc.PublicKey) if err != nil { return "", err } } - diskId, err := self.host.zone.region.ReplaceSystemDisk(self.InstanceId, imageId, passwd, keypair, sysSizeGB) + diskId, err := self.host.zone.region.ReplaceSystemDisk(self.InstanceId, desc.ImageId, desc.Password, keypair, desc.SysSizeGB) if err != nil { return "", err } diff --git a/pkg/multicloud/aws/instance.go b/pkg/multicloud/aws/instance.go index d7bccf4ea8..bcb88b5428 100644 --- a/pkg/multicloud/aws/instance.go +++ b/pkg/multicloud/aws/instance.go @@ -404,7 +404,7 @@ func (self *SInstance) UpdateVM(ctx context.Context, name string) error { return self.host.zone.region.UpdateVM(self.InstanceId, name) } -func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { udata, err := self.GetUserData() if err != nil { return "", err @@ -424,30 +424,30 @@ func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd s keypairName := self.KeyPairName loginUser := cloudinit.NewUser(api.VM_AWS_DEFAULT_LOGIN_USER) loginUser.SudoPolicy(cloudinit.USER_SUDO_NOPASSWD) - if len(publicKey) > 0 { - loginUser.SshKey(publicKey) + if len(desc.PublicKey) > 0 { + loginUser.SshKey(desc.PublicKey) cloudconfig.MergeUser(loginUser) - keypairName, err = self.host.zone.region.syncKeypair(publicKey) + keypairName, err = self.host.zone.region.syncKeypair(desc.PublicKey) if err != nil { return "", fmt.Errorf("RebuildRoot.syncKeypair %s", err) } - } else if len(passwd) > 0 { - loginUser.Password(passwd) + } else if len(desc.Password) > 0 { + loginUser.Password(desc.Password) cloudconfig.MergeUser(loginUser) } // compare sysSizeGB - image, err := self.host.zone.region.GetImage(imageId) + image, err := self.host.zone.region.GetImage(desc.ImageId) if err != nil { return "", err } else { minSizeGB := image.GetMinOsDiskSizeGb() - if minSizeGB > sysSizeGB { - sysSizeGB = minSizeGB + if minSizeGB > desc.SysSizeGB { + desc.SysSizeGB = minSizeGB } } - diskId, err := self.host.zone.region.ReplaceSystemDisk(ctx, self.InstanceId, imageId, sysSizeGB, keypairName, cloudconfig.UserDataBase64()) + diskId, err := self.host.zone.region.ReplaceSystemDisk(ctx, self.InstanceId, desc.ImageId, desc.SysSizeGB, keypairName, cloudconfig.UserDataBase64()) if err != nil { return "", err } diff --git a/pkg/multicloud/azure/classic_instance.go b/pkg/multicloud/azure/classic_instance.go index c1f4a89322..6f21e741e9 100644 --- a/pkg/multicloud/azure/classic_instance.go +++ b/pkg/multicloud/azure/classic_instance.go @@ -324,7 +324,7 @@ func (self *SClassicInstance) DeployVM(ctx context.Context, name string, usernam //return self.host.zone.region.DeployVM(self.ID, name, password, publicKey, deleteKeypair, description) } -func (self *SClassicInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SClassicInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { return "", cloudprovider.ErrNotImplemented //return self.host.zone.region.ReplaceSystemDisk(self.ID, imageId, passwd, publicKey, int32(sysSizeGB)) } diff --git a/pkg/multicloud/azure/instance.go b/pkg/multicloud/azure/instance.go index 8b7b5ac9b8..b08006f42f 100644 --- a/pkg/multicloud/azure/instance.go +++ b/pkg/multicloud/azure/instance.go @@ -775,11 +775,11 @@ func (region *SRegion) DeployVM(ctx context.Context, instanceId, name, password, return nil } -func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { cpu := self.GetVcpuCount() memoryMb := self.GetVmemSizeMB() self.StopVM(ctx, true) - return self.host.zone.region.ReplaceSystemDisk(self, cpu, memoryMb, imageId, passwd, publicKey, sysSizeGB) + return self.host.zone.region.ReplaceSystemDisk(self, cpu, memoryMb, desc.ImageId, desc.Password, desc.PublicKey, desc.SysSizeGB) } func (region *SRegion) ReplaceSystemDisk(instance *SInstance, cpu int, memoryMb int, imageId, passwd, publicKey string, sysSizeGB int) (string, error) { diff --git a/pkg/multicloud/ctyun/instance.go b/pkg/multicloud/ctyun/instance.go index 57d6220847..625ef951c2 100644 --- a/pkg/multicloud/ctyun/instance.go +++ b/pkg/multicloud/ctyun/instance.go @@ -429,7 +429,7 @@ func (self *SInstance) UpdateUserData(userData string) error { return cloudprovider.ErrNotImplemented } -func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { return "", cloudprovider.ErrNotImplemented } diff --git a/pkg/multicloud/esxi/virtualmachine.go b/pkg/multicloud/esxi/virtualmachine.go index 029343d995..2600de7ce1 100644 --- a/pkg/multicloud/esxi/virtualmachine.go +++ b/pkg/multicloud/esxi/virtualmachine.go @@ -159,7 +159,7 @@ func (self *SVirtualMachine) DeployVM(ctx context.Context, name string, username return cloudprovider.ErrNotImplemented } -func (self *SVirtualMachine) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SVirtualMachine) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { return "", cloudprovider.ErrNotImplemented } @@ -1078,3 +1078,7 @@ func (self *SVirtualMachine) ExportTemplate(ctx context.Context, idx int, diskPa log.Debugf("download to %s finish", diskPath) return nil } + +func (self *SVirtualMachine) GetSerialOutput(port int) (string, error) { + return "", cloudprovider.ErrNotImplemented +} diff --git a/pkg/multicloud/google/bucket.go b/pkg/multicloud/google/bucket.go new file mode 100644 index 0000000000..68c3848ee1 --- /dev/null +++ b/pkg/multicloud/google/bucket.go @@ -0,0 +1,149 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "fmt" + "io" + "net/http" + "net/url" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SLifecycleRuleAction struct { + Type string +} + +type SLifecycleRuleCondition struct { + Age int +} + +type SLifecycleRule struct { + Action SLifecycleRuleAction + Condition SLifecycleRuleCondition +} + +type SBucketPolicyOnly struct { + Enabled bool +} + +type SUniformBucketLevelAccess struct { + Enabled bool +} + +type SIamConfiguration struct { + BucketPolicyOnly SBucketPolicyOnly + UniformBucketLevelAccess SUniformBucketLevelAccess +} + +type SLifecycle struct { + Rule []SLifecycleRule +} + +type SBucket struct { + Kind string + SelfLink string + Id string + Name string + ProjectNumber string + Metageneration string + Location string + StorageClass string + Etag string + TimeCreated time.Time + Updated time.Time + Lifecycle SLifecycle + IamConfiguration SIamConfiguration + LocationType string +} + +func (region *SRegion) GetBucket(name string) (*SBucket, error) { + resource := "b/" + name + bucket := &SBucket{} + err := region.StorageGet(resource, bucket) + if err != nil { + return nil, errors.Wrap(err, "GetBucket") + } + return bucket, nil +} + +func (region *SRegion) GetBuckets(maxResults int, pageToken string) ([]SBucket, error) { + buckets := []SBucket{} + params := map[string]string{ + "project": region.GetProjectId(), + } + err := region.StorageList("b", params, maxResults, pageToken, &buckets) + if err != nil { + return nil, err + } + return buckets, nil +} + +func (region *SRegion) CreateBucket(name string, storageClass string) (*SBucket, error) { + body := map[string]interface{}{ + "name": name, + "location": region.Name, + } + if len(storageClass) > 0 { + body["storageClass"] = storageClass + } + params := url.Values{} + params.Set("project", region.GetProjectId()) + bucket := &SBucket{} + err := region.StorageInsert(fmt.Sprintf("b?%s", params.Encode()), jsonutils.Marshal(body), bucket) + if err != nil { + return nil, err + } + return bucket, nil +} + +func (region *SRegion) UploadObject(bucket string, params url.Values, header http.Header, input io.Reader) error { + resource := fmt.Sprintf("b/%s/o", bucket) + if len(params) > 0 { + resource = fmt.Sprintf("%s?%s", resource, params.Encode()) + } + return region.client.storageUpload(resource, header, input) +} + +func (region *SRegion) PutObject(bucket string, name string, input io.Reader, contType string, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType) error { + params := url.Values{} + params.Set("name", name) + params.Set("uploadType", "media") + switch cannedAcl { + case cloudprovider.ACLPrivate: + params.Set("predefinedAcl", "private") + case cloudprovider.ACLAuthRead: + params.Set("predefinedAcl", "authenticatedRead") + case cloudprovider.ACLPublicRead: + params.Set("predefinedAcl", "publicRead") + case cloudprovider.ACLPublicReadWrite: + return cloudprovider.ErrNotSupported + } + + header := http.Header{} + header.Set("Content-Length", fmt.Sprintf("%v", sizeBytes)) + header.Set("Content-Type", "application/octet-stream") + + return region.UploadObject(bucket, params, header, input) +} + +func (region *SRegion) DeleteBucket(name string) error { + return region.StorageDelete("b/" + name) +} diff --git a/pkg/multicloud/google/cloudbuild.go b/pkg/multicloud/google/cloudbuild.go new file mode 100644 index 0000000000..702747c4ff --- /dev/null +++ b/pkg/multicloud/google/cloudbuild.go @@ -0,0 +1,41 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import "yunion.io/x/pkg/errors" + +type SCloudbuildBuild struct { + Id string + Status string + LogUrl string +} + +type SCloudbuildMetadata struct { + Build SCloudbuildBuild +} + +type SCloudbuildOperation struct { + Name string + Metadata SCloudbuildMetadata +} + +func (region *SRegion) GetCloudbuildOperation(name string) (*SCloudbuildOperation, error) { + operation := SCloudbuildOperation{} + err := region.cloudbuildGet(name, &operation) + if err != nil { + return nil, errors.Wrap(err, "region.cloudbuildGet") + } + return &operation, nil +} diff --git a/pkg/multicloud/google/disk.go b/pkg/multicloud/google/disk.go index 60dbafce1f..14269c4fd2 100644 --- a/pkg/multicloud/google/disk.go +++ b/pkg/multicloud/google/disk.go @@ -43,6 +43,7 @@ type SDisk struct { LabelFingerprint string PhysicalBlockSizeBytes string ResourcePolicies []string + Users []string Kind string autoDelete bool boot bool @@ -116,6 +117,9 @@ func (disk *SDisk) GetDiskSizeMB() int { } func (disk *SDisk) GetIsAutoDelete() bool { + if len(disk.Users) == 0 { + return false + } return disk.autoDelete } @@ -124,7 +128,7 @@ func (disk *SDisk) GetTemplateId() string { } func (disk *SDisk) GetDiskType() string { - if disk.index == 0 || disk.boot { + if disk.boot && len(disk.Users) > 0 { return api.DISK_TYPE_SYS } return api.DISK_TYPE_DATA @@ -155,11 +159,16 @@ func (disk *SDisk) GetAccessPath() string { } func (disk *SDisk) Delete(ctx context.Context) error { - return cloudprovider.ErrNotImplemented + return disk.storage.zone.region.Delete(disk.SelfLink) } func (disk *SDisk) CreateISnapshot(ctx context.Context, name string, desc string) (cloudprovider.ICloudSnapshot, error) { - return nil, cloudprovider.ErrNotImplemented + snapshot, err := disk.storage.zone.region.CreateSnapshot(disk.SelfLink, name, desc) + if err != nil { + return nil, err + } + snapshot.region = disk.storage.zone.region + return snapshot, nil } func (disk *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { @@ -189,15 +198,15 @@ func (disk *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { } func (disk *SDisk) Resize(ctx context.Context, newSizeMB int64) error { - return cloudprovider.ErrNotImplemented + return disk.storage.zone.region.ResizeDisk(disk.SelfLink, int(newSizeMB>>10)) } func (disk *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) { - return "", cloudprovider.ErrNotImplemented + return "", cloudprovider.ErrNotSupported } func (disk *SDisk) Rebuild(ctx context.Context) error { - return cloudprovider.ErrNotImplemented + return cloudprovider.ErrNotSupported } func (disk *SDisk) GetBillingType() string { @@ -215,3 +224,47 @@ func (disk *SDisk) GetExpiredAt() time.Time { func (disk *SDisk) GetProjectId() string { return disk.storage.zone.region.GetProjectId() } + +func (region *SRegion) CreateDisk(name string, sizeGb int, zone string, storageType string, image string, desc string) (*SDisk, error) { + if !strings.HasPrefix(storageType, GOOGLE_COMPUTE_DOMAIN) { + storageType = fmt.Sprintf("projects/%s/zones/%s/diskTypes/%s", region.GetProjectId(), zone, storageType) + } + body := map[string]interface{}{ + "name": name, + "description": desc, + // https://www.googleapis.com/compute/v1/projects/my-project-15390453537169/zones/us-west2-c/diskTypes/pd-standard + // projects/my-project-15390453537169/zones/us-west2-c/diskTypes/pd-standard + "type": storageType, + } + if len(image) > 0 { + body["sourceImage"] = image + } else { + body["sizeGb"] = sizeGb + } + disk := &SDisk{} + resource := fmt.Sprintf("zones/%s/disks", zone) + err := region.Insert(resource, jsonutils.Marshal(body), disk) + if err != nil { + return nil, err + } + return disk, nil +} + +func (region *SRegion) ResizeDisk(id string, sizeGb int) error { + body := map[string]int{ + "sizeGb": sizeGb, + } + return region.Do(id, "resize", nil, jsonutils.Marshal(body)) +} + +func (region *SRegion) CreateSnapshot(diskId string, name string, desc string) (*SSnapshot, error) { + body := map[string]string{ + "name": name, + "description": desc, + } + err := region.Do(diskId, "createSnapshot", nil, jsonutils.Marshal(body)) + if err != nil { + return nil, err + } + return region.GetSnapshot(fmt.Sprintf("projects/%s/global/snapshots/%s", region.GetProjectId(), name)) +} diff --git a/pkg/multicloud/google/eip.go b/pkg/multicloud/google/eip.go index 5523d67e75..e838d460a6 100644 --- a/pkg/multicloud/google/eip.go +++ b/pkg/multicloud/google/eip.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" billing "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -145,17 +146,71 @@ func (addr *SAddress) GetInternetChargeType() string { } func (addr *SAddress) Delete() error { - return cloudprovider.ErrNotImplemented + return addr.region.Delete(addr.SelfLink) } func (addr *SAddress) Associate(instanceId string) error { - return cloudprovider.ErrNotImplemented + return addr.region.AssociateInstanceEip(instanceId, addr.Address) } func (addr *SAddress) Dissociate() error { - return cloudprovider.ErrNotImplemented + if len(addr.Users) > 0 { + return addr.region.DissociateInstanceEip(addr.Users[0], addr.Address) + } + return nil } func (addr *SAddress) ChangeBandwidth(bw int) error { - return cloudprovider.ErrNotImplemented + return cloudprovider.ErrNotSupported +} + +func (region *SRegion) CreateEip(name string, desc string) (*SAddress, error) { + body := map[string]string{ + "name": name, + "description": desc, + } + resource := fmt.Sprintf("regions/%s/addresses", region.Name) + addr := &SAddress{region: region} + err := region.Insert(resource, jsonutils.Marshal(body), addr) + if err != nil { + return nil, err + } + return addr, nil +} + +func (region *SRegion) AssociateInstanceEip(instanceId string, eip string) error { + instance, err := region.GetInstance(instanceId) + if err != nil { + return errors.Wrap(err, "region.GetInstance") + } + for _, networkInterface := range instance.NetworkInterfaces { + body := map[string]interface{}{ + "type": "ONE_TO_ONE_NAT", + "name": "External NAT", + "natIP": eip, + } + params := map[string]string{"networkInterface": networkInterface.Name} + return region.Do(instance.SelfLink, "addAccessConfig", params, jsonutils.Marshal(body)) + } + return fmt.Errorf("no valid networkinterface to associate") +} + +func (region *SRegion) DissociateInstanceEip(instanceId string, eip string) error { + instance, err := region.GetInstance(instanceId) + if err != nil { + return errors.Wrap(err, "region.GetInstance") + } + for _, networkInterface := range instance.NetworkInterfaces { + for _, accessConfig := range networkInterface.AccessConfigs { + if accessConfig.NatIP == eip { + body := map[string]string{} + params := map[string]string{ + "networkInterface": networkInterface.Name, + "accessConfig": accessConfig.Name, + } + return region.Do(instance.SelfLink, "deleteAccessConfig", params, jsonutils.Marshal(body)) + } + } + } + return nil } diff --git a/pkg/multicloud/google/globalnetwork.go b/pkg/multicloud/google/globalnetwork.go index 7df59e683b..128e031ef4 100644 --- a/pkg/multicloud/google/globalnetwork.go +++ b/pkg/multicloud/google/globalnetwork.go @@ -17,6 +17,7 @@ package google import ( "time" + "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" ) @@ -60,3 +61,16 @@ func (cli *SGoogleClient) GetGlobalNetworks(maxResults int, pageToken string) ([ } return networks, nil } + +func (region *SRegion) CreateGlobalNetwork(name string, desc string) (*SGlobalNetwork, error) { + body := map[string]string{ + "name": name, + "description": desc, + } + globalnetwork := &SGlobalNetwork{} + err := region.Insert("global/networks", jsonutils.Marshal(body), globalnetwork) + if err != nil { + return nil, errors.Wrap(err, "region.Insert") + } + return globalnetwork, nil +} diff --git a/pkg/multicloud/google/google.go b/pkg/multicloud/google/google.go index bffb16e6c2..6b8c2cdae2 100644 --- a/pkg/multicloud/google/google.go +++ b/pkg/multicloud/google/google.go @@ -17,15 +17,18 @@ package google import ( "context" "fmt" + "io" "net/http" "net/url" "strings" + "unicode" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "golang.org/x/oauth2/jwt" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -39,11 +42,21 @@ const ( GOOGLE_DEFAULT_REGION = "asia-east1" - GOOGLE_COMPUTE_DOMAIN = "https://www.googleapis.com/compute" - GOOGLE_MANAGER_DOMAIN = "https://cloudresourcemanager.googleapis.com" - GOOGLE_API_VERSION = "v1" GOOGLE_MANAGER_API_VERSION = "v1" + + GOOGLE_STORAGE_API_VERSION = "v1" + GOOGLE_CLOUDBUILD_API_VERSION = "v1" + GOOGLE_BILLING_API_VERSION = "v1" + + GOOGLE_MANAGER_DOMAIN = "https://cloudresourcemanager.googleapis.com" + GOOGLE_COMPUTE_DOMAIN = "https://www.googleapis.com/compute" + GOOGLE_STORAGE_DOMAIN = "https://storage.googleapis.com/storage" + GOOGLE_CLOUDBUILD_DOMAIN = "https://cloudbuild.googleapis.com" + GOOGLE_STORAGE_UPLOAD_DOMAIN = "https://www.googleapis.com/upload/storage" + GOOGLE_BILLING_DOMAIN = "https://cloudbilling.googleapis.com" + + MAX_RETRY = 3 ) type SGoogleClient struct { @@ -85,6 +98,9 @@ func NewGoogleClient(providerId string, providerName string, projectId, clientEm "https://www.googleapis.com/auth/cloud-platform.read-only", "https://www.googleapis.com/auth/cloudplatformprojects", "https://www.googleapis.com/auth/cloudplatformprojects.readonly", + + "https://www.googleapis.com/auth/devstorage.full_control", + "https://www.googleapis.com/auth/devstorage.read_write", }, TokenURL: google.JWTTokenURL, } @@ -113,6 +129,9 @@ func (self *SGoogleClient) fetchRegions() error { func jsonRequest(client *http.Client, method httputils.THttpMethod, domain, apiVersion, resource string, params map[string]string, body jsonutils.JSONObject, debug bool) (jsonutils.JSONObject, error) { resource = strings.TrimPrefix(resource, fmt.Sprintf("%s/%s/", domain, apiVersion)) + if len(resource) == 0 { + return nil, cloudprovider.ErrNotFound + } _url := fmt.Sprintf("%s/%s/%s", domain, apiVersion, resource) values := url.Values{} for k, v := range params { @@ -127,7 +146,7 @@ func jsonRequest(client *http.Client, method httputils.THttpMethod, domain, apiV func (self *SGoogleClient) ecsGet(resource string, retval interface{}) error { resp, err := jsonRequest(self.client, "GET", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, nil, nil, self.Debug) if err != nil { - return errors.Wrap(err, "jsonRequest") + return err } if retval != nil { err = resp.Unmarshal(retval) @@ -179,10 +198,247 @@ func (self *SGoogleClient) ecsListAll(resource string, params map[string]string, return items.Unmarshal(retval) } -func _jsonRequest(client *http.Client, method httputils.THttpMethod, url string, body jsonutils.JSONObject, debug bool) (jsonutils.JSONObject, error) { - _, data, err := httputils.JSONRequest(client, context.Background(), method, url, nil, body, debug) +func (self *SGoogleClient) ecsDelete(id string, retval interface{}) error { + resp, err := jsonRequest(self.client, "DELETE", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, id, nil, nil, self.Debug) if err != nil { - if strings.Index(err.Error(), "not found") > 0 { + return err + } + if retval != nil { + return resp.Unmarshal(retval) + } + return nil +} + +func (self *SGoogleClient) ecsPatch(resource string, action string, params map[string]string, body jsonutils.JSONObject) (string, error) { + if len(action) > 0 { + resource = fmt.Sprintf("%s/%s", resource, action) + } + resp, err := jsonRequest(self.client, "PATCH", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, body, self.Debug) + if err != nil { + return "", err + } + selfLink, _ := resp.GetString("selfLink") + return selfLink, nil +} + +func (self *SGoogleClient) ecsDo(resource string, action string, params map[string]string, body jsonutils.JSONObject) (string, error) { + resource = fmt.Sprintf("%s/%s", resource, action) + resp, err := jsonRequest(self.client, "POST", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, body, self.Debug) + if err != nil { + return "", err + } + selfLink, _ := resp.GetString("selfLink") + return selfLink, nil +} + +func (self *SGoogleClient) ecsInsert(resource string, body jsonutils.JSONObject, retval interface{}) error { + resource = fmt.Sprintf("projects/%s/%s", self.projectId, resource) + if name, _ := body.GetString("name"); len(name) > 0 { + generateName := "" + for _, s := range name { + if unicode.IsLetter(s) || unicode.IsDigit(s) { + generateName = fmt.Sprintf("%s%c", generateName, s) + } else { + generateName = fmt.Sprintf("%s-", generateName) + } + } + if name != generateName { + err := jsonutils.Update(body, map[string]string{"name": generateName}) + if err != nil { + log.Errorf("faild to generate google name from %s -> %s", name, generateName) + } + } + } + resp, err := jsonRequest(self.client, "POST", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, nil, body, self.Debug) + if err != nil { + return err + } + if retval != nil { + return resp.Unmarshal(retval) + } + return nil +} + +func (self *SGoogleClient) storageInsert(resource string, body jsonutils.JSONObject, retval interface{}) error { + resp, err := jsonRequest(self.client, "POST", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, nil, body, self.Debug) + if err != nil { + return err + } + if retval != nil { + return resp.Unmarshal(retval) + } + return nil +} + +func (self *SGoogleClient) storageUpload(resource string, header http.Header, body io.Reader) error { + return rawRequest(self.client, "POST", GOOGLE_STORAGE_UPLOAD_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, header, body, self.Debug) +} + +func (self *SGoogleClient) storageList(resource string, params map[string]string) (jsonutils.JSONObject, error) { + return jsonRequest(self.client, "GET", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, params, nil, self.Debug) +} + +func (self *SGoogleClient) storageListAll(resource string, params map[string]string, retval interface{}) error { + if params == nil { + params = map[string]string{} + } + items := jsonutils.NewArray() + nextPageToken := "" + params["maxResults"] = "500" + for { + params["pageToken"] = nextPageToken + resp, err := self.storageList(resource, params) + if err != nil { + return errors.Wrap(err, "storageList") + } + if resp.Contains("items") { + _items, err := resp.GetArray("items") + if err != nil { + return errors.Wrap(err, "resp.GetArray") + } + items.Add(_items...) + } + nextPageToken, _ = resp.GetString("nextPageToken") + if len(nextPageToken) == 0 { + break + } + } + return items.Unmarshal(retval) +} + +func (self *SGoogleClient) storageGet(resource string, retval interface{}) error { + resp, err := jsonRequest(self.client, "GET", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, nil, nil, self.Debug) + if err != nil { + return err + } + if retval != nil { + err = resp.Unmarshal(retval) + if err != nil { + return errors.Wrap(err, "resp.Unmarshal") + } + } + return nil +} + +func (self *SGoogleClient) storageDelete(id string, retval interface{}) error { + resp, err := jsonRequest(self.client, "DELETE", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, id, nil, nil, self.Debug) + if err != nil { + return err + } + if retval != nil { + return resp.Unmarshal(retval) + } + return nil +} + +func (self *SGoogleClient) storageDo(resource string, action string, params map[string]string, body jsonutils.JSONObject) (string, error) { + resource = fmt.Sprintf("%s/%s", resource, action) + resp, err := jsonRequest(self.client, "POST", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, params, body, self.Debug) + if err != nil { + return "", err + } + selfLink, _ := resp.GetString("selfLink") + return selfLink, nil +} + +func (self *SGoogleClient) cloudbuildGet(resource string, retval interface{}) error { + resp, err := jsonRequest(self.client, "GET", GOOGLE_CLOUDBUILD_DOMAIN, GOOGLE_CLOUDBUILD_API_VERSION, resource, nil, nil, self.Debug) + if err != nil { + return err + } + if retval != nil { + err = resp.Unmarshal(retval) + if err != nil { + return errors.Wrap(err, "resp.Unmarshal") + } + } + return nil +} + +func (self *SGoogleClient) cloudbuildInsert(resource string, body jsonutils.JSONObject, retval interface{}) error { + resp, err := jsonRequest(self.client, "POST", GOOGLE_CLOUDBUILD_DOMAIN, GOOGLE_CLOUDBUILD_API_VERSION, resource, nil, body, self.Debug) + if err != nil { + return err + } + if retval != nil { + return resp.Unmarshal(retval) + } + return nil +} + +func (self *SGoogleClient) billingList(resource string, params map[string]string) (jsonutils.JSONObject, error) { + return jsonRequest(self.client, "GET", GOOGLE_BILLING_DOMAIN, GOOGLE_BILLING_API_VERSION, resource, params, nil, self.Debug) +} + +func (self *SGoogleClient) billingListAll(resource string, params map[string]string, retval interface{}) error { + if params == nil { + params = map[string]string{} + } + items := jsonutils.NewArray() + nextPageToken := "" + params["pageSize"] = "5000" + for { + params["pageToken"] = nextPageToken + resp, err := self.billingList(resource, params) + if err != nil { + return errors.Wrap(err, "billingList") + } + if resp.Contains("skus") { + _items, err := resp.GetArray("skus") + if err != nil { + return errors.Wrap(err, "resp.GetArray") + } + items.Add(_items...) + } + nextPageToken, _ = resp.GetString("nextPageToken") + if len(nextPageToken) == 0 { + break + } + } + return items.Unmarshal(retval) +} + +func rawRequest(client *http.Client, method httputils.THttpMethod, domain, apiVersion string, resource string, header http.Header, body io.Reader, debug bool) error { + resource = strings.TrimPrefix(resource, fmt.Sprintf("%s/%s/", domain, apiVersion)) + resource = fmt.Sprintf("%s/%s/%s", domain, apiVersion, resource) + _, err := httputils.Request(client, context.Background(), method, resource, header, body, debug) + return err +} + +func _jsonRequest(client *http.Client, method httputils.THttpMethod, url string, body jsonutils.JSONObject, debug bool) (jsonutils.JSONObject, error) { + var ( + retry bool = false + err error = nil + data jsonutils.JSONObject = nil + ) + for i := 0; i < MAX_RETRY; i++ { + _, data, err = httputils.JSONRequest(client, context.Background(), method, url, nil, body, debug) + if err != nil { + if body != nil { + log.Errorf("%s %s params: %s error: %v", method, url, body.PrettyString(), err) + } else { + log.Errorf("%s %s error: %v", method, url, err) + } + for _, msg := range []string{ + "EOF", + "i/o timeout", + "TLS handshake timeout", + } { + if strings.Index(err.Error(), msg) >= 0 { + retry = true + break + } + } + if !retry { + break + } + } + if !retry { + break + } + } + if err != nil { + if strings.Index(strings.ToLower(err.Error()), "not found") > 0 { return nil, cloudprovider.ErrNotFound } return nil, errors.Wrap(err, "JSONRequest") diff --git a/pkg/multicloud/google/host.go b/pkg/multicloud/google/host.go index 86806ebf0a..dc77e01678 100644 --- a/pkg/multicloud/google/host.go +++ b/pkg/multicloud/google/host.go @@ -167,11 +167,17 @@ func (host *SHost) GetIVMById(id string) (cloudprovider.ICloudVM, error) { if instance.Zone != host.zone.SelfLink { return nil, cloudprovider.ErrNotFound } + instance.host = host return instance, nil } func (host *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) { - return nil, cloudprovider.ErrNotImplemented + instance, err := host.zone.region._createVM(host.zone.Name, desc) + if err != nil { + return nil, err + } + instance.host = host + return instance, nil } func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { diff --git a/pkg/multicloud/google/image.go b/pkg/multicloud/google/image.go index 4a6d759218..08151b7c20 100644 --- a/pkg/multicloud/google/image.go +++ b/pkg/multicloud/google/image.go @@ -16,6 +16,7 @@ package google import ( "context" + "fmt" "strings" "time" @@ -187,7 +188,7 @@ func (image *SImage) GetCreatedAt() time.Time { } func (image *SImage) GetImageFormat() string { - return "vhd" + return "raw" } func (image *SImage) IsEmulated() bool { @@ -213,3 +214,32 @@ func (region *SRegion) fetchImages() ([]SImage, error) { region.client.images = images return images, nil } + +func (region *SRegion) CreateImage(name string, desc string, bucketName string, sourceFile string) (*SImage, error) { + body := map[string]interface{}{ + "timeout": "7200s", + "steps": []struct { + Args []string + Name string + }{ + { + Args: []string{ + fmt.Sprintf("-source_file=gs://%s/%s", bucketName, sourceFile), + "-data_disk", + "-timeout=7056s", + "-image_name=" + name, + "-no_guest_environment", + "-client_id=onecloud", + "-description=" + desc, + }, + Name: "gcr.io/compute-image-tools/gce_vm_image_import:release", + }, + }, + "tags": []string{"gce-daisy", "gce-daisy-image-import"}, + } + err := region.CloudbuildInsert(jsonutils.Marshal(body)) + if err != nil { + return nil, err + } + return region.GetImage(fmt.Sprintf("projects/%s/global/images/%s", region.GetProjectId(), name)) +} diff --git a/pkg/multicloud/google/instance.go b/pkg/multicloud/google/instance.go index 24d8994684..2d12571ab2 100644 --- a/pkg/multicloud/google/instance.go +++ b/pkg/multicloud/google/instance.go @@ -21,7 +21,9 @@ import ( "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/utils" @@ -30,9 +32,17 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/cloudinit" "yunion.io/x/onecloud/pkg/util/imagetools" ) +const ( + METADATA_SSH_KEYS = "ssh-keys" + METADATA_STARTUP_SCRIPT = "startup-script" + METADATA_POWER_SHELL = "sysprep-specialize-script-ps1" + METADATA_STARTUP_SCRIPT_POWER_SHELL = "windows-startup-script-ps1" +) + type AccessConfig struct { Type string Name string @@ -65,6 +75,16 @@ type SInstanceTag struct { Fingerprint string } +type SMetadataItem struct { + Key string + Value string +} + +type SMetadata struct { + Fingerprint string + Items []SMetadataItem +} + type SInstance struct { multicloud.SInstanceBase host *SHost @@ -80,7 +100,7 @@ type SInstance struct { CanIpForward bool NetworkInterfaces []SNetworkInterface Disks []InstanceDisk - Metadata map[string]string + Metadata SMetadata ServiceAccounts []ServiceAccount Scheduling map[string]interface{} CpuPlatform string @@ -153,7 +173,7 @@ func (instance *SInstance) GetStatus() string { case "SUSPENDED": return api.VM_SUSPEND case "TERMINATED": - return api.VM_DELETING + return api.VM_READY default: return api.VM_UNKNOWN } @@ -186,17 +206,20 @@ func (instance *SInstance) GetIHostId() string { func (instance *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { idisks := []cloudprovider.ICloudDisk{} for _, disk := range instance.Disks { - disk, err := instance.host.zone.region.GetDisk(disk.Source) + _disk, err := instance.host.zone.region.GetDisk(disk.Source) if err != nil { return nil, errors.Wrap(err, "GetDisk") } - storage, err := instance.host.zone.region.GetStorage(disk.Type) + storage, err := instance.host.zone.region.GetStorage(_disk.Type) if err != nil { return nil, errors.Wrap(err, "GetStorage") } storage.zone = instance.host.zone - disk.storage = storage - idisks = append(idisks, disk) + _disk.storage = storage + _disk.autoDelete = disk.AutoDelete + _disk.boot = disk.Boot + _disk.index = disk.Index + idisks = append(idisks, _disk) } return idisks, nil } @@ -219,6 +242,7 @@ func (instance *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { return nil, errors.Wrapf(err, "region.GetEip(%s)", conf.NatIP) } if len(eips) == 1 { + eips[0].region = instance.host.zone.region return &eips[0], nil } eip := &SAddress{ @@ -297,7 +321,29 @@ func (instance *SInstance) GetInstanceType() string { } func (instance *SInstance) AssignSecurityGroup(id string) error { - return cloudprovider.ErrNotImplemented + for _, secgrpType := range []string{SECGROUP_TYPE_TAG, SECGROUP_TYPE_SERVICE_ACCOUNT} { + if strings.Contains(id, fmt.Sprintf("/%s/", secgrpType)) { + idx := strings.LastIndex(id, "/") + 1 + if idx <= 0 { + return fmt.Errorf("invalid secgroup %s with id %s", secgrpType, id) + } + secgroup := id[idx:] + switch secgrpType { + case SECGROUP_TYPE_TAG: + tag := strings.ToLower(secgroup) + if !utils.IsInStringArray(tag, instance.Tags.Items) { + instance.Tags.Items = append(instance.Tags.Items, tag) + return instance.host.zone.region.SetTags(instance.SelfLink, instance.Tags) + } + case SECGROUP_TYPE_SERVICE_ACCOUNT: + if len(instance.ServiceAccounts) > 0 { + return fmt.Errorf("instance %s has already set serviceAccount %s", instance.Name, instance.ServiceAccounts[0].Email) + } + return instance.host.zone.region.SetServiceAccount(instance.SelfLink, secgroup) + } + } + } + return fmt.Errorf("unknown secgroup type %s", id) } func (instance *SInstance) GetSecurityGroupIds() ([]string, error) { @@ -318,14 +364,14 @@ func (instance *SInstance) GetSecurityGroupIds() ([]string, error) { if len(instance.ServiceAccounts) > 0 && isecgroup.GetName() == instance.ServiceAccounts[0].Email { secgroupIds = append(secgroupIds, isecgroup.GetGlobalId()) } - if isecgroup.GetName() == globalnetwork.Name { + if isecgroup.GetName() == globalnetwork.Name && !strings.Contains(isecgroup.GetGlobalId(), fmt.Sprintf("/%s/", SECGROUP_TYPE_TAG)) { secgroupIds = append(secgroupIds, isecgroup.GetGlobalId()) } } } if len(instance.NetworkInterfaces) == 1 { for _, secgroup := range isecgroups { - if utils.IsInStringArray(secgroup.GetName(), instance.Tags.Items) { + if utils.IsInStringArray(secgroup.GetName(), instance.Tags.Items) && strings.Contains(secgroup.GetGlobalId(), fmt.Sprintf("/%s/", SECGROUP_TYPE_TAG)) { secgroupIds = append(secgroupIds, secgroup.GetGlobalId()) } } @@ -334,7 +380,46 @@ func (instance *SInstance) GetSecurityGroupIds() ([]string, error) { } func (instance *SInstance) SetSecurityGroups(ids []string) error { - return cloudprovider.ErrNotImplemented + secgroups := map[string][]string{} + for _, id := range ids { + for _, secgrpType := range []string{SECGROUP_TYPE_TAG, SECGROUP_TYPE_SERVICE_ACCOUNT} { + if strings.Contains(id, fmt.Sprintf("/%s/", secgrpType)) { + idx := strings.LastIndex(id, "/") + 1 + if idx <= 0 { + return fmt.Errorf("invalid secgroup %s with id %s", secgrpType, id) + } + secgroup := id[idx:] + if len(secgroup) == 0 { + return fmt.Errorf("invalid secgroup %s with id %s", secgrpType, id) + } + if _, ok := secgroups[secgrpType]; !ok { + secgroups[secgrpType] = []string{} + } + if !utils.IsInStringArray(secgroup, secgroups[secgrpType]) { + secgroups[secgrpType] = append(secgroups[secgrpType], secgroup) + } + } + } + } + if tags, ok := secgroups[SECGROUP_TYPE_TAG]; ok && len(tags) > 0 { + for _, tag := range tags { + tag = strings.ToLower(tag) + if !utils.IsInStringArray(tag, instance.Tags.Items) { + instance.Tags.Items = append(instance.Tags.Items, tag) + } + } + err := instance.host.zone.region.SetTags(instance.SelfLink, instance.Tags) + if err != nil { + return errors.Wrap(err, "SetTags") + } + } + if serviceAccounts, ok := secgroups[SECGROUP_TYPE_SERVICE_ACCOUNT]; ok && len(serviceAccounts) > 0 { + if len(serviceAccounts) > 1 { + return fmt.Errorf("can not set multi service account for google instance") + } + return instance.host.zone.region.SetServiceAccount(instance.SelfLink, serviceAccounts[0]) + } + return nil } func (instance *SInstance) GetHypervisor() string { @@ -342,46 +427,90 @@ func (instance *SInstance) GetHypervisor() string { } func (instance *SInstance) StartVM(ctx context.Context) error { - return cloudprovider.ErrNotImplemented + return instance.host.zone.region.StartInstance(instance.SelfLink) } func (instance *SInstance) StopVM(ctx context.Context, isForce bool) error { - return cloudprovider.ErrNotImplemented + return instance.host.zone.region.StopInstance(instance.SelfLink) } func (instance *SInstance) DeleteVM(ctx context.Context) error { - return cloudprovider.ErrNotImplemented + return instance.host.zone.region.Delete(instance.SelfLink) } func (instance *SInstance) UpdateVM(ctx context.Context, name string) error { - return cloudprovider.ErrNotImplemented + return cloudprovider.ErrNotSupported } func (instance *SInstance) UpdateUserData(userData string) error { - return cloudprovider.ErrNotImplemented + items := []SMetadataItem{} + for _, item := range instance.Metadata.Items { + if item.Key != METADATA_STARTUP_SCRIPT && item.Key != METADATA_POWER_SHELL && item.Key != METADATA_STARTUP_SCRIPT_POWER_SHELL { + items = append(items, item) + } + } + if len(userData) > 0 { + items = append(items, SMetadataItem{Key: METADATA_STARTUP_SCRIPT, Value: userData}) + items = append(items, SMetadataItem{Key: METADATA_STARTUP_SCRIPT_POWER_SHELL, Value: userData}) + items = append(items, SMetadataItem{Key: METADATA_POWER_SHELL, Value: userData}) + } + instance.Metadata.Items = items + return instance.host.zone.region.SetMetadata(instance.SelfLink, instance.Metadata) } -func (instance *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { - return "", cloudprovider.ErrNotImplemented +func (instance *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { + diskId, err := instance.host.zone.region.RebuildRoot(instance.SelfLink, desc.ImageId, desc.SysSizeGB) + if err != nil { + return "", errors.Wrap(err, "region.RebuildRoot") + } + return diskId, instance.DeployVM(ctx, "", desc.Account, desc.Password, desc.PublicKey, false, "") } func (instance *SInstance) DeployVM(ctx context.Context, name string, username string, password string, publicKey string, deleteKeypair bool, description string) error { - return cloudprovider.ErrNotImplemented + conf := cloudinit.SCloudConfig{} + user := cloudinit.NewUser(username) + if len(password) > 0 { + user.Password(password) + } + if len(publicKey) > 0 { + user.SshKey(publicKey) + } + if len(password) > 0 || len(publicKey) > 0 { + conf.MergeUser(user) + items := []SMetadataItem{} + instance.Refresh() + for _, item := range instance.Metadata.Items { + if item.Key != METADATA_STARTUP_SCRIPT_POWER_SHELL && item.Key != METADATA_STARTUP_SCRIPT { + items = append(items, item) + } + } + items = append(items, SMetadataItem{Key: METADATA_STARTUP_SCRIPT_POWER_SHELL, Value: conf.UserDataPowerShell()}) + items = append(items, SMetadataItem{Key: METADATA_STARTUP_SCRIPT, Value: conf.UserDataScript()}) + instance.Metadata.Items = items + return instance.host.zone.region.SetMetadata(instance.SelfLink, instance.Metadata) + } + return nil } func (instance *SInstance) ChangeConfig(ctx context.Context, config *cloudprovider.SManagedVMChangeConfig) error { - return cloudprovider.ErrNotImplemented + return instance.host.zone.region.ChangeInstanceConfig(instance.SelfLink, instance.host.zone.Name, config.InstanceType, config.Cpu, config.MemoryMB) } func (instance *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { return nil, cloudprovider.ErrNotImplemented } + func (instance *SInstance) AttachDisk(ctx context.Context, diskId string) error { - return cloudprovider.ErrNotImplemented + return instance.host.zone.region.AttachDisk(instance.SelfLink, diskId, false) } func (instance *SInstance) DetachDisk(ctx context.Context, diskId string) error { - return cloudprovider.ErrNotImplemented + for _, disk := range instance.Disks { + if strings.HasSuffix(disk.Source, diskId) { + return instance.host.zone.region.DetachDisk(instance.SelfLink, disk.DeviceName) + } + } + return nil } func (instance *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { @@ -389,9 +518,363 @@ func (instance *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid stri } func (instance *SInstance) Renew(bc billing.SBillingCycle) error { - return cloudprovider.ErrNotImplemented + return cloudprovider.ErrNotSupported } func (instance *SInstance) GetError() error { return nil } + +func getDiskInfo(disk string) (cloudprovider.SDiskInfo, error) { + result := cloudprovider.SDiskInfo{} + diskInfo := strings.Split(disk, ":") + for _, d := range diskInfo { + if utils.IsInStringArray(d, []string{api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_LOCAL_SSD}) { + result.StorageType = d + } else if memSize, err := fileutils.GetSizeMb(d, 'M', 1024); err == nil { + result.SizeGB = memSize >> 10 + } else { + result.Name = d + } + } + if len(result.StorageType) == 0 { + result.StorageType = api.STORAGE_GOOGLE_PD_STANDARD + } + + if result.SizeGB == 0 { + return result, fmt.Errorf("Missing disk size") + } + return result, nil +} + +func (region *SRegion) CreateInstance(zone, name, desc, instanceType string, cpu, memoryMb int, networkId string, ipAddr, imageId string, disks []string) (*SInstance, error) { + if len(instanceType) == 0 && (cpu == 0 || memoryMb == 0) { + return nil, fmt.Errorf("Missing instanceType or cpu &memory info") + } + if len(disks) == 0 { + return nil, fmt.Errorf("Missing disk info") + } + sysDisk, err := getDiskInfo(disks[0]) + if err != nil { + return nil, errors.Wrap(err, "getDiskInfo.sys") + } + dataDisks := []cloudprovider.SDiskInfo{} + for _, d := range disks[1:] { + dataDisk, err := getDiskInfo(d) + if err != nil { + return nil, errors.Wrapf(err, "getDiskInfo(%s)", d) + } + dataDisks = append(dataDisks, dataDisk) + } + conf := &cloudprovider.SManagedVMCreateConfig{ + Name: name, + Description: desc, + ExternalImageId: imageId, + Cpu: cpu, + MemoryMB: memoryMb, + ExternalNetworkId: networkId, + IpAddr: ipAddr, + SysDisk: sysDisk, + DataDisks: dataDisks, + } + return region._createVM(zone, conf) +} + +func (region *SRegion) getSecgroupByIds(ids []string) (map[string][]string, error) { + secgroups := map[string][]string{} + for _, id := range ids { + for _, secgrpType := range []string{SECGROUP_TYPE_TAG, SECGROUP_TYPE_SERVICE_ACCOUNT} { + if strings.Contains(id, fmt.Sprintf("/%s/", secgrpType)) { + idx := strings.LastIndex(id, "/") + 1 + if idx <= 0 { + return nil, fmt.Errorf("invalid secgroup %s for %s", id, secgrpType) + } + secgroup := id[idx:] + if len(secgroup) == 0 { + return nil, fmt.Errorf("invalid secgroup %s for %s", id, secgrpType) + } + if _, ok := secgroups[secgrpType]; !ok { + secgroups[secgrpType] = []string{} + } + if !utils.IsInStringArray(secgroup, secgroups[secgrpType]) { + secgroups[secgrpType] = append(secgroups[secgrpType], secgroup) + } + } + } + } + return secgroups, nil +} + +func (region *SRegion) _createVM(zone string, desc *cloudprovider.SManagedVMCreateConfig) (*SInstance, error) { + network, err := region.GetNetwork(desc.ExternalNetworkId) + if err != nil { + return nil, errors.Wrap(err, "region.GetNetwork") + } + secgroups, err := region.getSecgroupByIds(desc.ExternalSecgroupIds) + if err != nil { + return nil, errors.Wrap(err, "getSecgroupByIds") + } + serviceAccounts, ok := secgroups[SECGROUP_TYPE_SERVICE_ACCOUNT] + if ok && len(serviceAccounts) > 1 { + return nil, fmt.Errorf("Security groups are distributed across multiple service accounts") + } + if len(desc.InstanceType) == 0 { + desc.InstanceType = fmt.Sprintf("custom-%d-%d", desc.Cpu, desc.MemoryMB) + } + disks := []map[string]interface{}{} + if len(desc.SysDisk.Name) == 0 { + desc.SysDisk.Name = fmt.Sprintf("vdisk-%s-%d", desc.Name, time.Now().UnixNano()) + } + disks = append(disks, map[string]interface{}{ + "boot": true, + "initializeParams": map[string]interface{}{ + "diskName": strings.Replace(desc.SysDisk.Name, "_", "-", -1), + "sourceImage": desc.ExternalImageId, + "diskSizeGb": desc.SysDisk.SizeGB, + "diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, desc.SysDisk.StorageType), + }, + "autoDelete": true, + }) + for _, disk := range desc.DataDisks { + if len(disk.Name) == 0 { + disk.Name = fmt.Sprintf("vdisk-%s-%d", desc.Name, time.Now().UnixNano()) + } + disks = append(disks, map[string]interface{}{ + "boot": false, + "initializeParams": map[string]interface{}{ + "diskName": strings.Replace(disk.Name, "_", "-", -1), + "diskSizeGb": disk.SizeGB, + "diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.StorageType), + }, + "autoDelete": true, + }) + } + networkInterface := map[string]string{ + "network": network.Network, + "subnetwork": network.SelfLink, + } + if len(desc.IpAddr) > 0 { + networkInterface["networkIp"] = desc.IpAddr + } + params := map[string]interface{}{ + "name": desc.Name, + "description": desc.Description, + "machineType": fmt.Sprintf("zones/%s/machineTypes/%s", zone, desc.InstanceType), + "networkInterfaces": []map[string]string{ + networkInterface, + }, + "disks": disks, + } + if tags, ok := secgroups[SECGROUP_TYPE_TAG]; ok && len(tags) > 0 { + for i := range tags { + tags[i] = strings.ToLower(tags[i]) + } + params["tags"] = map[string][]string{ + "items": tags, + } + } + if len(desc.UserData) > 0 { + params["metadata"] = map[string]interface{}{ + "items": []struct { + Key string + Value string + }{ + { + Key: METADATA_STARTUP_SCRIPT, + Value: desc.UserData, + }, + { + Key: METADATA_POWER_SHELL, + Value: desc.UserData, + }, + }, + } + } + if len(serviceAccounts) > 0 { + params["serviceAccounts"] = []struct { + Email string + Scopes []string + }{ + { + Email: serviceAccounts[0], + Scopes: []string{ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append", + }, + }, + } + } + log.Debugf("create google instance params: %s", jsonutils.Marshal(params).String()) + instance := &SInstance{} + resource := fmt.Sprintf("zones/%s/instances", zone) + err = region.Insert(resource, jsonutils.Marshal(params), instance) + if err != nil { + return nil, err + } + return instance, nil +} + +func (region *SRegion) StartInstance(id string) error { + params := map[string]string{} + return region.Do(id, "start", nil, jsonutils.Marshal(params)) +} + +func (region *SRegion) StopInstance(id string) error { + params := map[string]string{} + return region.Do(id, "stop", nil, jsonutils.Marshal(params)) +} + +func (region *SRegion) ResetInstance(id string) error { + params := map[string]string{} + return region.Do(id, "reset", nil, jsonutils.Marshal(params)) +} + +func (region *SRegion) DetachDisk(instanceId, deviceName string) error { + body := map[string]string{} + params := map[string]string{"deviceName": deviceName} + return region.Do(instanceId, "detachDisk", params, jsonutils.Marshal(body)) +} + +func (instance *SInstance) GetSerialOutput(port int) (string, error) { + return instance.host.zone.region.GetSerialPortOutput(instance.SelfLink, port) +} + +func (region *SRegion) GetSerialPortOutput(id string, port int) (string, error) { + _content, content, next := "", "", 0 + var err error = nil + for { + _content, next, err = region.getSerialPortOutput(id, port, next) + if err != nil { + return content, err + } + content += _content + if len(_content) == 0 { + break + } + } + return content, nil +} + +func (region *SRegion) getSerialPortOutput(id string, port int, start int) (string, int, error) { + resource := fmt.Sprintf("%s/serialPort?port=%d&start=%d", id, port, start) + result := struct { + Contents string + Start int + Next int + }{} + err := region.Get(resource, &result) + if err != nil { + return "", result.Next, errors.Wrap(err, "") + } + return result.Contents, result.Next, nil +} + +func (region *SRegion) AttachDisk(instanceId, diskId string, boot bool) error { + diskId = strings.TrimPrefix(diskId, fmt.Sprintf("%s/%s/", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION)) + diskId = fmt.Sprintf("%s/%s/%s", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, diskId) + body := map[string]interface{}{ + "source": diskId, + "boot": boot, + } + if boot { + body["autoDelete"] = true + } + params := map[string]string{"forceAttach": "true"} + return region.Do(instanceId, "attachDisk", params, jsonutils.Marshal(body)) +} + +func (region *SRegion) ChangeInstanceConfig(id string, zone string, instanceType string, cpu int, memoryMb int) error { + if len(instanceType) == 0 { + instanceType = fmt.Sprintf("custom-%d-%d", cpu, memoryMb) + } + params := map[string]string{ + "machineType": fmt.Sprintf("zones/%s/machineTypes/%s", zone, instanceType), + } + return region.Do(id, "setMachineType", nil, jsonutils.Marshal(params)) +} + +func (region *SRegion) SetMetadata(id string, metadata SMetadata) error { + return region.Do(id, "setMetadata", nil, jsonutils.Marshal(metadata)) +} + +func (region *SRegion) SetTags(id string, tags SInstanceTag) error { + return region.Do(id, "setTags", nil, jsonutils.Marshal(tags)) +} + +func (region *SRegion) SetServiceAccount(id string, email string) error { + body := map[string]interface{}{ + "email": email, + "scopes": []string{ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring.write", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append", + }, + } + return region.Do(id, "setsetServiceAccount", nil, jsonutils.Marshal(body)) +} + +func (region *SRegion) RebuildRoot(instanceId string, imageId string, sysDiskSizeGb int) (string, error) { + oldDisk, diskType, deviceName := "", api.STORAGE_GOOGLE_PD_STANDARD, "" + instance, err := region.GetInstance(instanceId) + if err != nil { + return "", errors.Wrap(err, "region.GetInstance") + } + for _, disk := range instance.Disks { + if disk.Boot { + oldDisk = disk.Source + deviceName = disk.DeviceName + break + } + } + + if len(oldDisk) > 0 { + disk, err := region.GetDisk(oldDisk) + if err != nil { + return "", errors.Wrap(err, "region.GetDisk") + } + diskType = disk.Type + if sysDiskSizeGb == 0 { + sysDiskSizeGb = disk.SizeGB + } + } + + zone, err := region.GetZone(instance.Zone) + if err != nil { + return "", errors.Wrap(err, "region.GetZone") + } + + diskName := fmt.Sprintf("vdisk-%s-%d", instance.Name, time.Now().UnixNano()) + disk, err := region.CreateDisk(diskName, sysDiskSizeGb, zone.Name, diskType, imageId, "create for replace instance system disk") + if err != nil { + return "", errors.Wrap(err, "region.CreateDisk.systemDisk") + } + + if len(deviceName) > 0 { + err = region.DetachDisk(instanceId, deviceName) + if err != nil { + defer region.Delete(disk.SelfLink) + return "", errors.Wrap(err, "region.DetachDisk") + } + } + + err = region.AttachDisk(instanceId, disk.SelfLink, true) + if err != nil { + if len(oldDisk) > 0 { + defer region.AttachDisk(instanceId, oldDisk, true) + } + defer region.Delete(disk.SelfLink) + return "", errors.Wrap(err, "region.AttachDisk.newSystemDisk") + } + + if len(oldDisk) > 0 { + defer region.Delete(oldDisk) + } + return disk.GetGlobalId(), nil +} diff --git a/pkg/multicloud/google/network.go b/pkg/multicloud/google/network.go index 26eba1e7ce..85a78dfad0 100644 --- a/pkg/multicloud/google/network.go +++ b/pkg/multicloud/google/network.go @@ -84,7 +84,7 @@ func (network *SNetwork) GetStatus() string { } func (network *SNetwork) Delete() error { - return cloudprovider.ErrNotImplemented + return network.wire.vpc.region.Delete(network.SelfLink) } func (network *SNetwork) GetAllocTimeoutSeconds() int { @@ -129,3 +129,19 @@ func (network *SNetwork) GetIsPublic() bool { func (network *SNetwork) GetPublicScope() rbacutils.TRbacScope { return rbacutils.ScopeDomain } + +func (region *SRegion) CreateNetwork(name string, vpc string, cidr string, desc string) (*SNetwork, error) { + body := map[string]interface{}{ + "name": name, + "description": desc, + "network": vpc, + "ipCidrRange": cidr, + } + resource := fmt.Sprintf("regions/%s/subnetworks", region.Name) + network := &SNetwork{} + err := region.Insert(resource, jsonutils.Marshal(body), network) + if err != nil { + return nil, err + } + return network, nil +} diff --git a/pkg/multicloud/google/operation.go b/pkg/multicloud/google/operation.go new file mode 100644 index 0000000000..544209c03e --- /dev/null +++ b/pkg/multicloud/google/operation.go @@ -0,0 +1,71 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "time" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +const ( + OPERATION_STATUS_RUNNING = "RUNNING" + OPERATION_STATUS_DONE = "DONE" +) + +type SOperation struct { + Id string + Name string + OperationType string + TargetLink string + TargetId string + Status string + User string + Progress int + InsertTime time.Time + StartTime time.Time + EndTime time.Time + SelfLink string + Region string + Kind string +} + +func (region *SRegion) GetOperation(id string) (*SOperation, error) { + operation := &SOperation{} + err := region.Get(id, &operation) + if err != nil { + return nil, err + } + return operation, nil +} + +func (region *SRegion) WaitOperation(id string, resource, action string) (string, error) { + targetLink := "" + err := cloudprovider.Wait(time.Second*5, time.Minute*5, func() (bool, error) { + operation, err := region.GetOperation(id) + if err != nil { + return false, err + } + log.Debugf("%s %s operation status: %s expect %s", action, resource, operation.Status, OPERATION_STATUS_DONE) + if operation.Status == OPERATION_STATUS_DONE { + targetLink = operation.TargetLink + return true, nil + } + return false, nil + }) + return targetLink, err +} diff --git a/pkg/multicloud/google/region.go b/pkg/multicloud/google/region.go index 72cd706d52..e825b74b01 100644 --- a/pkg/multicloud/google/region.go +++ b/pkg/multicloud/google/region.go @@ -20,6 +20,7 @@ import ( "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" @@ -140,6 +141,45 @@ func (region *SRegion) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { return nil, cloudprovider.ErrNotFound } +func (region *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) { + globalnetwork, err := region.CreateGlobalNetwork(name, desc) + if err != nil { + return nil, errors.Wrap(err, "region.CreateGlobalNetwork") + } + vpc := &SVpc{region: region, globalnetwork: globalnetwork} + return vpc, nil +} + +func (region *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + storage, err := region.GetStorage(id) + if err != nil { + return nil, err + } + zone, err := region.GetZone(storage.Zone) + if err != nil { + return nil, errors.Wrapf(err, "region.GetZone(%s)", storage.Zone) + } + zone.region = region + storage.zone = zone + return storage, nil +} + +func (self *SRegion) GetIHostById(id string) (cloudprovider.ICloudHost, error) { + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + ihost, err := izones[i].GetIHostById(id) + if err == nil { + return ihost, nil + } else if err != cloudprovider.ErrNotFound { + return nil, err + } + } + return nil, cloudprovider.ErrNotFound +} + func (region *SRegion) GetProjectId() string { return region.client.projectId } @@ -240,6 +280,32 @@ func (region *SRegion) GetISnapshotById(id string) (cloudprovider.ICloudSnapshot return snapshot, nil } +func (region *SRegion) BillingList(resource string, params map[string]string, pageSize int, pageToken string, retval interface{}) error { + if pageSize == 0 && len(pageToken) == 0 { + return region.BillingListAll(resource, params, retval) + } + if params == nil { + params = map[string]string{} + } + params["pageSize"] = fmt.Sprintf("%d", pageSize) + params["pageToken"] = pageToken + resp, err := region.client.billingList(resource, params) + if err != nil { + return errors.Wrap(err, "billingList") + } + if resp.Contains("skus") && retval != nil { + err = resp.Unmarshal(retval, "skus") + if err != nil { + return errors.Wrap(err, "resp.Unmarshal") + } + } + return nil +} + +func (region *SRegion) BillingListAll(resource string, params map[string]string, retval interface{}) error { + return region.client.billingListAll(resource, params, retval) +} + func (region *SRegion) ListAll(resource string, params map[string]string, retval interface{}) error { return region.client.ecsListAll(resource, params, retval) } @@ -270,6 +336,140 @@ func (region *SRegion) Get(id string, retval interface{}) error { return region.client.ecsGet(id, retval) } +func (region *SRegion) StorageListAll(resource string, params map[string]string, retval interface{}) error { + return region.client.storageListAll(resource, params, retval) +} + +func (region *SRegion) StorageList(resource string, params map[string]string, maxResults int, pageToken string, retval interface{}) error { + if maxResults == 0 && len(pageToken) == 0 { + return region.client.storageListAll(resource, params, retval) + } + if params == nil { + params = map[string]string{} + } + params["maxResults"] = fmt.Sprintf("%d", maxResults) + params["pageToken"] = pageToken + resp, err := region.client.storageList(resource, params) + if err != nil { + return errors.Wrap(err, "storageList") + } + if resp.Contains("items") && retval != nil { + err = resp.Unmarshal(retval, "items") + if err != nil { + return errors.Wrap(err, "resp.Unmarshal") + } + } + return nil +} + +func (region *SRegion) StorageGet(id string, retval interface{}) error { + return region.client.storageGet(id, retval) +} + +func (region *SRegion) StorageDo(id string, action string, params map[string]string, body jsonutils.JSONObject) error { + opId, err := region.client.storageDo(id, action, params, body) + if err != nil { + return err + } + if strings.Index(opId, "/operations/") > 0 { + _, err = region.WaitOperation(opId, id, action) + return err + } + return nil +} + +func (region *SRegion) Do(id string, action string, params map[string]string, body jsonutils.JSONObject) error { + opId, err := region.client.ecsDo(id, action, params, body) + if err != nil { + return err + } + if strings.Index(opId, "/operations/") > 0 { + _, err = region.WaitOperation(opId, id, action) + return err + } + return nil +} + +func (region *SRegion) Patch(id string, action string, params map[string]string, body jsonutils.JSONObject) error { + opId, err := region.client.ecsPatch(id, action, params, body) + if err != nil { + return err + } + if strings.Index(opId, "/operations/") > 0 { + _, err = region.WaitOperation(opId, id, action) + return err + } + return nil +} + +func (region *SRegion) StorageDelete(id string) error { + return region.client.storageDelete(id, nil) +} + +func (region *SRegion) Delete(id string) error { + operation := &SOperation{} + err := region.client.ecsDelete(id, operation) + if err != nil { + return errors.Wrap(err, "client.ecsDelete") + } + _, err = region.WaitOperation(operation.SelfLink, id, "delete") + if err != nil { + return errors.Wrapf(err, "region.WaitOperation(%s)", operation.SelfLink) + } + return nil +} + +func (region *SRegion) StorageInsert(resource string, body jsonutils.JSONObject, retval interface{}) error { + return region.client.storageInsert(resource, body, retval) +} + +func (region *SRegion) CloudbuildInsert(body jsonutils.JSONObject) error { + result := &struct { + Name string + }{} + resource := fmt.Sprintf("projects/%s/builds", region.GetProjectId()) + err := region.client.cloudbuildInsert(resource, body, result) + if err != nil { + return errors.Wrap(err, "insert") + } + err = cloudprovider.Wait(time.Second*10, time.Minute*40, func() (bool, error) { + operation, err := region.GetCloudbuildOperation(result.Name) + if err != nil { + return false, errors.Wrapf(err, "region.GetCloudbuildOperation(%s)", result.Name) + } + status := operation.Metadata.Build.Status + log.Debugf("cloudbuild %s status: %s", result.Name, status) + if status == "FAILURE" { + return false, fmt.Errorf("cloudbuild failed error log: %s", operation.Metadata.Build.LogUrl) + } + if status == "SUCCESS" { + return true, nil + } + return false, nil + }) + if err != nil { + return errors.Wrap(err, "cloudprovider.Wait") + } + return nil +} + +func (region *SRegion) cloudbuildGet(id string, retval interface{}) error { + return region.client.cloudbuildGet(id, retval) +} + +func (region *SRegion) Insert(resource string, body jsonutils.JSONObject, retval interface{}) error { + operation := &SOperation{} + err := region.client.ecsInsert(resource, body, operation) + if err != nil { + return err + } + resourceId, err := region.WaitOperation(operation.SelfLink, resource, "insert") + if err != nil { + return errors.Wrapf(err, "region.WaitOperation(%s)", operation.SelfLink) + } + return region.Get(resourceId, retval) +} + func (region *SRegion) fetchResourcePolicies() ([]SResourcePolicy, error) { if len(region.client.resourcepolices) > 0 { return region.client.resourcepolices, nil @@ -307,3 +507,11 @@ func (region *SRegion) GetISnapshotPolicyById(id string) (cloudprovider.ICloudSn } return policy, nil } + +func (region *SRegion) CreateEIP(args *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) { + eip, err := region.CreateEip(args.Name, "") + if err != nil { + return nil, err + } + return eip, nil +} diff --git a/pkg/multicloud/google/securitygroup.go b/pkg/multicloud/google/securitygroup.go index def647ef61..5ef572599e 100644 --- a/pkg/multicloud/google/securitygroup.go +++ b/pkg/multicloud/google/securitygroup.go @@ -31,6 +31,11 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" ) +const ( + SECGROUP_TYPE_SERVICE_ACCOUNT = "serviceAccount" + SECGROUP_TYPE_TAG = "tag" +) + type SFirewallAction struct { IPProtocol string Ports []string @@ -66,7 +71,7 @@ func (f FirewallSet) Swap(i, j int) { func (f FirewallSet) Less(i, j int) bool { if f[i].Priority != f[j].Priority { - return f[i].Priority < f[j].Priority + return f[i].Priority > f[j].Priority } return len(f[i].Allowed) < len(f[j].Allowed) } @@ -103,7 +108,7 @@ func (firewall *SFirewall) _toRules(action secrules.TSecurityRuleAction) ([]secr rule := secrules.SecurityRule{ Action: action, Direction: secrules.DIR_IN, - Description: firewall.Description, + Description: firewall.SelfLink, Priority: firewall.Priority, } if firewall.Direction == "EGRESS" { @@ -148,6 +153,12 @@ func (firewall *SFirewall) _toRules(action secrules.TSecurityRuleAction) ([]secr rule.PortEnd = -1 rules = append(rules, rule) } + if len(allow.Ports) == 0 { + rule.Ports = []int{} + rule.PortStart = -1 + rule.PortEnd = -1 + rules = append(rules, rule) + } } } return rules, nil @@ -174,10 +185,10 @@ func (secgroup *SSecurityGroup) GetId() string { func (secgroup *SSecurityGroup) GetGlobalId() string { if len(secgroup.Tag) > 0 { - return fmt.Sprintf("%s/%s", secgroup.GetId(), secgroup.Tag) + return fmt.Sprintf("%s/%s/%s", secgroup.GetId(), SECGROUP_TYPE_TAG, secgroup.Tag) } if len(secgroup.ServiceAccount) > 0 { - return fmt.Sprintf("%s/%s", secgroup.GetId(), secgroup.ServiceAccount) + return fmt.Sprintf("%s/%s/%s", secgroup.GetId(), SECGROUP_TYPE_SERVICE_ACCOUNT, secgroup.ServiceAccount) } return secgroup.GetId() } @@ -213,6 +224,16 @@ func (secgroup *SSecurityGroup) Refresh() error { } func (secgroup *SSecurityGroup) Delete() error { + rules, err := secgroup.GetRules() + if err != nil { + return errors.Wrap(err, "GetRules") + } + for _, rule := range rules { + err = secgroup.vpc.region.DeleteSecgroupRule(rule.Description, rule) + if err != nil { + return errors.Wrapf(err, "DeleteSecgroupRule(%s)", rule.Description) + } + } return nil } @@ -258,8 +279,100 @@ func (self *SSecurityGroup) GetRules() ([]secrules.SecurityRule, error) { return rules, nil } +func (region *SRegion) DeleteSecgroupRule(ruleId string, rule secrules.SecurityRule) error { + firwall, err := region.GetFirewall(ruleId) + if err != nil { + return errors.Wrap(err, "region.GetFirewall") + } + currentRule, err := firwall.toRules() + if err != nil { + return errors.Wrap(err, "firwall.toRules") + } + if len(currentRule) > 1 { + for _, _rule := range currentRule { + if _rule.String() != rule.String() { + for _, tag := range firwall.TargetTags { + err = region.CreateSecurityGroupRule(_rule, firwall.Network, tag, "") + if err != nil { + return errors.Wrap(err, "region.CreateSecurityGroupRule") + } + } + for _, serviceAccount := range firwall.TargetServiceAccounts { + err = region.CreateSecurityGroupRule(_rule, firwall.Network, "", serviceAccount) + if err != nil { + return errors.Wrap(err, "region.CreateSecurityGroupRule") + } + } + if len(firwall.TargetTags)+len(firwall.TargetServiceAccounts) == 0 { + err = region.CreateSecurityGroupRule(_rule, firwall.Network, "", "") + if err != nil { + return errors.Wrap(err, "region.CreateSecurityGroupRule") + } + } + } + } + } + return region.Delete(firwall.SelfLink) +} + func (secgroup *SSecurityGroup) SyncRules(rules []secrules.SecurityRule) error { - return cloudprovider.ErrNotImplemented + if len(rules) == 0 { + rules = append(rules, *secrules.MustParseSecurityRule("in:deny any")) + } + currentRule, err := secgroup.GetRules() + if err != nil { + return errors.Wrap(err, "secgroup.GetRules") + } + sort.Sort(secrules.SecurityRuleSet(rules)) + region := secgroup.vpc.region + deleteRules := map[string]secrules.SecurityRule{} + addRules := []secrules.SecurityRule{} + i, j := 0, 0 + for i < len(rules) || j < len(currentRule) { + if i < len(rules) && j < len(currentRule) { + currentRuleStr := currentRule[j].String() + ruleStr := rules[i].String() + cmp := strings.Compare(currentRuleStr, ruleStr) + if cmp == 0 { + i += 1 + j += 1 + } else if cmp > 0 { + // delete rule + deleteRules[currentRule[j].Description] = currentRule[j] + j += 1 + } else { + rules[i].Priority = 101 - rules[i].Priority + addRules = append(addRules, rules[i]) + i += 1 + } + } else if i >= len(rules) { + // delete rule + deleteRules[currentRule[j].Description] = currentRule[j] + j += 1 + } else if j >= len(currentRule) { + // add rule + rules[i].Priority = 101 - rules[i].Priority + addRules = append(addRules, rules[i]) + err = region.CreateSecurityGroupRule(rules[i], secgroup.vpc.globalnetwork.SelfLink, secgroup.Tag, secgroup.ServiceAccount) + if err != nil { + return errors.Wrapf(err, "region.CreateSecurityGroupRule(%s)", rules[i].String()) + } + i += 1 + } + } + for id, rule := range deleteRules { + err = region.DeleteSecgroupRule(id, rule) + if err != nil { + return errors.Wrapf(err, "DeleteSecgroupRule(%s)", id) + } + } + for _, rule := range addRules { + err = region.CreateSecurityGroupRule(rule, secgroup.vpc.globalnetwork.SelfLink, secgroup.Tag, secgroup.ServiceAccount) + if err != nil { + return errors.Wrapf(err, "CreateSecurityGroupRule(%s)", rule.String()) + } + } + return nil } func (region *SRegion) GetISecurityGroupById(id string) (cloudprovider.ICloudSecurityGroup, error) { @@ -293,13 +406,94 @@ func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloud return nil, errors.Wrap(err, "ivpc.GetISecurityGroups") } for _, secgroup := range secgroups { - if secgroup.GetName() == name { + if strings.ToLower(secgroup.GetName()) == strings.ToLower(name) { return secgroup, nil } } return nil, cloudprovider.ErrNotFound } -func (region *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreateInput) (cloudprovider.ICloudSecurityGroup, error) { - return region.GetISecurityGroupByName(conf.VpcId, "") +func (region *SRegion) CreateSecurityGroupRule(rule secrules.SecurityRule, vpcId string, tag string, serviceAccount string) error { + name := fmt.Sprintf("%s-%d", rule.String(), rule.Priority) + if len(tag) > 0 { + name = fmt.Sprintf("for-tag-%s-%s", tag, name) + } + if len(serviceAccount) > 0 { + name = fmt.Sprintf("for-service-account-%s-%s", serviceAccount, name) + } + + body := map[string]interface{}{ + "name": strings.ToLower(name), + "priority": rule.Priority, + "network": vpcId, + "direction": "INGRESS", + } + if len(tag) > 0 { + body["targetTags"] = []string{strings.ToLower(tag)} + } + if len(serviceAccount) > 0 { + body["targetServiceAccounts"] = []string{serviceAccount} + } + if rule.Direction == secrules.DIR_OUT { + body["direction"] = "EGRESS" + } else { + body["sourceRanges"] = []string{rule.IPNet.String()} + } + + protocol := string(rule.Protocol) + if protocol == secrules.PROTO_ANY { + protocol = "all" + } + + ports := []string{} + if len(rule.Ports) > 0 { + for _, port := range rule.Ports { + ports = append(ports, fmt.Sprintf("%d", port)) + } + } else if rule.PortStart > 0 && rule.PortEnd > 0 { + if rule.PortStart == rule.PortEnd { + ports = append(ports, fmt.Sprintf("%d", rule.PortStart)) + } else { + ports = append(ports, fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd)) + } + } + + actionInfo := []struct { + IPProtocol string + Ports []string + }{ + { + IPProtocol: protocol, + Ports: ports, + }, + } + + if rule.Action == secrules.SecurityRuleDeny { + body["denied"] = actionInfo + } else { + body["allowed"] = actionInfo + } + + firwall := &SFirewall{} + err := region.Insert("global/firewalls", jsonutils.Marshal(body), firwall) + if err != nil { + if strings.Index(err.Error(), "already exists") >= 0 { + return nil + } + return errors.Wrap(err, "region.Insert") + } + return nil +} + +func (region *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreateInput) (cloudprovider.ICloudSecurityGroup, error) { + conf.VpcId = fmt.Sprintf("%s/%s", region.GetGlobalId(), conf.VpcId) + ivpc, err := region.GetIVpcById(conf.VpcId) + if err != nil { + return nil, errors.Wrapf(err, "region.GetIVpcById(%s)", conf.VpcId) + } + + vpc := ivpc.(*SVpc) + + secgroup := &SSecurityGroup{vpc: vpc, Tag: strings.ToLower(conf.Name)} + return secgroup, nil } diff --git a/pkg/multicloud/google/shell/bucket.go b/pkg/multicloud/google/shell/bucket.go new file mode 100644 index 0000000000..9e47a9353e --- /dev/null +++ b/pkg/multicloud/google/shell/bucket.go @@ -0,0 +1,68 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/google" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type BucketListOptions struct { + MaxResults int + PageToken string + } + + shellutils.R(&BucketListOptions{}, "bucket-list", "List buckets", func(cli *google.SRegion, args *BucketListOptions) error { + buckets, err := cli.GetBuckets(args.MaxResults, args.PageToken) + if err != nil { + return err + } + printList(buckets, 0, 0, 0, nil) + return nil + }) + + type BucketCreateOptions struct { + NAME string + StorageClass string `choices:"STANDARD|NEARLINE|COLDLINE"` + } + + shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create buckets", func(cli *google.SRegion, args *BucketCreateOptions) error { + bucket, err := cli.CreateBucket(args.NAME, args.StorageClass) + if err != nil { + return err + } + printObject(bucket) + return nil + }) + + type BucketNameOptions struct { + NAME string + } + + shellutils.R(&BucketNameOptions{}, "bucket-show", "Show bucket", func(cli *google.SRegion, args *BucketNameOptions) error { + bucket, err := cli.GetBucket(args.NAME) + if err != nil { + return err + } + printObject(bucket) + return nil + }) + + shellutils.R(&BucketNameOptions{}, "bucket-delete", "Delete bucket", func(cli *google.SRegion, args *BucketNameOptions) error { + return cli.DeleteBucket(args.NAME) + }) + +} diff --git a/pkg/multicloud/google/shell/cloudbuild.go b/pkg/multicloud/google/shell/cloudbuild.go new file mode 100644 index 0000000000..4c62b62f58 --- /dev/null +++ b/pkg/multicloud/google/shell/cloudbuild.go @@ -0,0 +1,35 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/google" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type CloudbuildOperationShowOptions struct { + NAME string + } + + shellutils.R(&CloudbuildOperationShowOptions{}, "cloud-build-operation-show", "Show cloudbuild operation", func(cli *google.SRegion, args *CloudbuildOperationShowOptions) error { + operation, err := cli.GetCloudbuildOperation(args.NAME) + if err != nil { + return err + } + printObject(operation) + return nil + }) +} diff --git a/pkg/multicloud/google/shell/disk.go b/pkg/multicloud/google/shell/disk.go index 3950c0ae0e..7e818220f4 100644 --- a/pkg/multicloud/google/shell/disk.go +++ b/pkg/multicloud/google/shell/disk.go @@ -35,10 +35,10 @@ func init() { return nil }) - type DiskShowOptions struct { + type DiskIdOptions struct { ID string } - shellutils.R(&DiskShowOptions{}, "disk-show", "Show disk", func(cli *google.SRegion, args *DiskShowOptions) error { + shellutils.R(&DiskIdOptions{}, "disk-show", "Show disk", func(cli *google.SRegion, args *DiskIdOptions) error { disk, err := cli.GetDisk(args.ID) if err != nil { return err @@ -47,6 +47,37 @@ func init() { return nil }) + shellutils.R(&DiskIdOptions{}, "disk-delete", "Delete disk", func(cli *google.SRegion, args *DiskIdOptions) error { + return cli.Delete(args.ID) + }) + + type DiskCreateOptions struct { + NAME string + Desc string + ZONE string + SIZE_GB int + Image string + STORAGE_TYPE string `choices:"pd-standard|pd-ssd"` + } + + shellutils.R(&DiskCreateOptions{}, "disk-create", "Create disks", func(cli *google.SRegion, args *DiskCreateOptions) error { + disk, err := cli.CreateDisk(args.NAME, args.SIZE_GB, args.ZONE, args.STORAGE_TYPE, args.Image, args.Desc) + if err != nil { + return err + } + printObject(disk) + return nil + }) + + type DiskResizeOptions struct { + ID string + SIZE_GB int + } + + shellutils.R(&DiskResizeOptions{}, "disk-resize", "Resize disk", func(cli *google.SRegion, args *DiskResizeOptions) error { + return cli.ResizeDisk(args.ID, args.SIZE_GB) + }) + type RegionDiskListOptions struct { StorageType string MaxResults int diff --git a/pkg/multicloud/google/shell/eip.go b/pkg/multicloud/google/shell/eip.go index e0b0ee2157..f1f58bd99f 100644 --- a/pkg/multicloud/google/shell/eip.go +++ b/pkg/multicloud/google/shell/eip.go @@ -34,10 +34,10 @@ func init() { return nil }) - type EipShowOptions struct { + type EipIdOptions struct { ID string } - shellutils.R(&EipShowOptions{}, "eip-show", "Show eip", func(cli *google.SRegion, args *EipShowOptions) error { + shellutils.R(&EipIdOptions{}, "eip-show", "Show eip", func(cli *google.SRegion, args *EipIdOptions) error { eip, err := cli.GetEip(args.ID) if err != nil { return err @@ -46,4 +46,22 @@ func init() { return nil }) + shellutils.R(&EipIdOptions{}, "eip-delete", "Delete eip", func(cli *google.SRegion, args *EipIdOptions) error { + return cli.Delete(args.ID) + }) + + type EipCreateOptions struct { + NAME string + Desc string + } + + shellutils.R(&EipCreateOptions{}, "eip-create", "Create eip", func(cli *google.SRegion, args *EipCreateOptions) error { + eip, err := cli.CreateEip(args.NAME, args.Desc) + if err != nil { + return err + } + printObject(eip) + return nil + }) + } diff --git a/pkg/multicloud/google/shell/globalnetwork.go b/pkg/multicloud/google/shell/globalnetwork.go index 6f09c9e288..496436baf5 100644 --- a/pkg/multicloud/google/shell/globalnetwork.go +++ b/pkg/multicloud/google/shell/globalnetwork.go @@ -45,4 +45,18 @@ func init() { return nil }) + type GlobalNetworkCreateOptions struct { + NAME string + Desc string + } + + shellutils.R(&GlobalNetworkCreateOptions{}, "global-network-create", "Create globalnetwork", func(cli *google.SRegion, args *GlobalNetworkCreateOptions) error { + globalnetwork, err := cli.CreateGlobalNetwork(args.NAME, args.Desc) + if err != nil { + return err + } + printObject(globalnetwork) + return nil + }) + } diff --git a/pkg/multicloud/google/shell/image.go b/pkg/multicloud/google/shell/image.go index fb39860bef..ff3cf9d197 100644 --- a/pkg/multicloud/google/shell/image.go +++ b/pkg/multicloud/google/shell/image.go @@ -21,7 +21,7 @@ import ( func init() { type ImageListOptions struct { - Project string + Project string `choices:"centos-cloud|ubuntu-os-cloud|windows-cloud|windows-sql-cloud|suse-cloud|suse-sap-cloud|rhel-cloud|rhel-sap-cloud|cos-cloud|debian-cloud"` MaxResults int PageToken string } @@ -46,4 +46,19 @@ func init() { return nil }) + type ImageCreateOptions struct { + NAME string + Desc string + BUCKET string + FILE string + } + + shellutils.R(&ImageCreateOptions{}, "image-create", "Create image", func(cli *google.SRegion, args *ImageCreateOptions) error { + image, err := cli.CreateImage(args.NAME, args.Desc, args.BUCKET, args.FILE) + if err != nil { + return err + } + printObject(image) + return nil + }) } diff --git a/pkg/multicloud/google/shell/instance.go b/pkg/multicloud/google/shell/instance.go index ece07ded15..3a6caacdaf 100644 --- a/pkg/multicloud/google/shell/instance.go +++ b/pkg/multicloud/google/shell/instance.go @@ -15,6 +15,10 @@ package shell import ( + "fmt" + + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/multicloud/google" "yunion.io/x/onecloud/pkg/util/shellutils" ) @@ -34,10 +38,10 @@ func init() { return nil }) - type InstanceShowOptions struct { + type InstanceIdOptions struct { ID string } - shellutils.R(&InstanceShowOptions{}, "instance-show", "Show instance", func(cli *google.SRegion, args *InstanceShowOptions) error { + shellutils.R(&InstanceIdOptions{}, "instance-show", "Show instance", func(cli *google.SRegion, args *InstanceIdOptions) error { instance, err := cli.GetInstance(args.ID) if err != nil { return err @@ -46,4 +50,138 @@ func init() { return nil }) + shellutils.R(&InstanceIdOptions{}, "instance-delete", "Delete instance", func(cli *google.SRegion, args *InstanceIdOptions) error { + return cli.Delete(args.ID) + }) + + shellutils.R(&InstanceIdOptions{}, "instance-start", "Start instance", func(cli *google.SRegion, args *InstanceIdOptions) error { + return cli.StartInstance(args.ID) + }) + + shellutils.R(&InstanceIdOptions{}, "instance-stop", "Stop instance", func(cli *google.SRegion, args *InstanceIdOptions) error { + return cli.StopInstance(args.ID) + }) + + shellutils.R(&InstanceIdOptions{}, "instance-reset", "Reset instance", func(cli *google.SRegion, args *InstanceIdOptions) error { + return cli.ResetInstance(args.ID) + }) + + type InstanceEipOptions struct { + ID string + EIP string `help:"eip address"` + } + + shellutils.R(&InstanceEipOptions{}, "instance-dissociate-eip", "Dissociate instance eip", func(cli *google.SRegion, args *InstanceEipOptions) error { + return cli.DissociateInstanceEip(args.ID, args.EIP) + }) + + shellutils.R(&InstanceEipOptions{}, "instance-associate-eip", "Associate instance eip", func(cli *google.SRegion, args *InstanceEipOptions) error { + return cli.AssociateInstanceEip(args.ID, args.EIP) + }) + + type InstanceDetachDiskOptions struct { + ID string + DeviceName string + } + + shellutils.R(&InstanceDetachDiskOptions{}, "instance-detach-disk", "Detach instance disk", func(cli *google.SRegion, args *InstanceDetachDiskOptions) error { + return cli.DetachDisk(args.ID, args.DeviceName) + }) + + type InstanceSetPublicKeyOptions struct { + ID string + PublicKey string + } + + shellutils.R(&InstanceSetPublicKeyOptions{}, "instance-set-publickey", "Set instance public key", func(cli *google.SRegion, args *InstanceSetPublicKeyOptions) error { + instance, err := cli.GetInstance(args.ID) + if err != nil { + return errors.Wrap(err, "cli.GetInstance") + } + items := []google.SMetadataItem{} + for _, item := range instance.Metadata.Items { + if item.Key != google.METADATA_SSH_KEYS { + items = append(items, item) + } + } + if len(args.PublicKey) > 0 { + items = append(items, google.SMetadataItem{Key: google.METADATA_SSH_KEYS, Value: "root:" + args.PublicKey}) + } + instance.Metadata.Items = items + return cli.SetMetadata(args.ID, instance.Metadata) + }) + + type InstanceAttachDiskOptions struct { + ID string + DISK string + Boot bool + } + + type InstanceSerialOutput struct { + ID string + PORT int + } + + shellutils.R(&InstanceSerialOutput{}, "instance-serial-output", "Get instance serial output", func(cli *google.SRegion, args *InstanceSerialOutput) error { + content, err := cli.GetSerialPortOutput(args.ID, args.PORT) + if err != nil { + return err + } + fmt.Printf("content: %s\n", content) + return nil + }) + + shellutils.R(&InstanceAttachDiskOptions{}, "instance-attach-disk", "Attach instance disk", func(cli *google.SRegion, args *InstanceAttachDiskOptions) error { + return cli.AttachDisk(args.ID, args.DISK, args.Boot) + }) + + type InstanceRebuildRootOptions struct { + ID string + IMAGE string + DiskSizeGb int + } + + shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Rebuild instance root", func(cli *google.SRegion, args *InstanceRebuildRootOptions) error { + diskId, err := cli.RebuildRoot(args.ID, args.IMAGE, args.DiskSizeGb) + if err != nil { + return err + } + fmt.Println(diskId) + return nil + }) + + type InstanceChangeConfigOptions struct { + ID string + ZONE string + InstanceType string + Cpu int + MemoryMb int + } + + shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Change instance config", func(cli *google.SRegion, args *InstanceChangeConfigOptions) error { + return cli.ChangeInstanceConfig(args.ID, args.ZONE, args.InstanceType, args.Cpu, args.MemoryMb) + }) + + type InstanceCreateOptions struct { + NAME string + ZONE string + IMAGE string + InstanceType string + Cpu int + MemoryMb int + NETWORK string + IpAddr string + Desc string + DISKS []string `nargs:"+"` + } + + shellutils.R(&InstanceCreateOptions{}, "instance-create", "Create instance", func(cli *google.SRegion, args *InstanceCreateOptions) error { + instance, err := cli.CreateInstance(args.ZONE, args.NAME, args.Desc, args.InstanceType, args.Cpu, args.MemoryMb, args.NETWORK, args.IpAddr, args.IMAGE, args.DISKS) + if err != nil { + return err + } + printObject(instance) + return nil + }) + } diff --git a/pkg/multicloud/google/shell/network.go b/pkg/multicloud/google/shell/network.go index db1931df21..455cff9d30 100644 --- a/pkg/multicloud/google/shell/network.go +++ b/pkg/multicloud/google/shell/network.go @@ -34,10 +34,11 @@ func init() { return nil }) - type NetworkShowOptions struct { + type NetworkIdOptions struct { ID string } - shellutils.R(&NetworkShowOptions{}, "network-show", "Show network", func(cli *google.SRegion, args *NetworkShowOptions) error { + + shellutils.R(&NetworkIdOptions{}, "network-show", "Show network", func(cli *google.SRegion, args *NetworkIdOptions) error { network, err := cli.GetNetwork(args.ID) if err != nil { return err @@ -46,4 +47,24 @@ func init() { return nil }) + shellutils.R(&NetworkIdOptions{}, "network-delete", "Delete network", func(cli *google.SRegion, args *NetworkIdOptions) error { + return cli.Delete(args.ID) + }) + + type NetworkCreateOptions struct { + NAME string + VPC string + CIDR string + Desc string + } + + shellutils.R(&NetworkCreateOptions{}, "network-create", "Create network", func(cli *google.SRegion, args *NetworkCreateOptions) error { + network, err := cli.CreateNetwork(args.NAME, args.VPC, args.CIDR, args.Desc) + if err != nil { + return err + } + printObject(network) + return nil + }) + } diff --git a/pkg/multicloud/google/shell/object.go b/pkg/multicloud/google/shell/object.go new file mode 100644 index 0000000000..78a3f1d83d --- /dev/null +++ b/pkg/multicloud/google/shell/object.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "os" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/google" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type ObjectPutOptions struct { + BUCKET string + FILE string + ContentType string + Acl string `choices:"private|public-read|public-read-write|authenticated-read"` + } + + shellutils.R(&ObjectPutOptions{}, "object-put", "Put object to buckets", func(cli *google.SRegion, args *ObjectPutOptions) error { + file, err := os.Open(args.FILE) + if err != nil { + return errors.Wrap(err, "so.Open") + } + stat, err := file.Stat() + if err != nil { + return errors.Wrap(err, "file.Stat") + } + return cli.PutObject(args.BUCKET, args.FILE, file, args.ContentType, stat.Size(), cloudprovider.TBucketACLType(args.Acl)) + }) + +} diff --git a/pkg/multicloud/google/shell/skubilling.go b/pkg/multicloud/google/shell/skubilling.go new file mode 100644 index 0000000000..febdf955cd --- /dev/null +++ b/pkg/multicloud/google/shell/skubilling.go @@ -0,0 +1,82 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "fmt" + "io/ioutil" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/multicloud/google" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type SkuBillingListOptions struct { + PageSize int + PageToken string + } + shellutils.R(&SkuBillingListOptions{}, "sku-billing-list", "List sku billing", func(cli *google.SRegion, args *SkuBillingListOptions) error { + billings, err := cli.ListSkuBilling(args.PageSize, args.PageToken) + if err != nil { + return err + } + printList(billings, 0, 0, 0, nil) + return nil + }) + + shellutils.R(&SkuBillingListOptions{}, "compute-sku-billing-list", "List sku billing", func(cli *google.SRegion, args *SkuBillingListOptions) error { + billings, err := cli.ListSkuBilling(args.PageSize, args.PageToken) + if err != nil { + return err + } + info := cli.GetSkuRateInfo(billings) + fmt.Println(jsonutils.Marshal(info).PrettyString()) + return nil + }) + + type SkuEstimate struct { + RATE_FAILE string + SKU string + REGION string + CPU int + MEMORY_MB int + } + + shellutils.R(&SkuEstimate{}, "sku-estimate", "Estimate sku price", func(cli *google.SRegion, args *SkuEstimate) error { + data, err := ioutil.ReadFile(args.RATE_FAILE) + if err != nil { + return errors.Wrap(err, "ioutil.ReadFile") + } + rate := google.SRateInfo{} + j, err := jsonutils.Parse(data) + if err != nil { + return errors.Wrap(err, "jsonutils.Parse") + } + err = jsonutils.Update(&rate, j) + if err != nil { + return errors.Wrap(err, "jsonutils.Update") + } + result, err := rate.GetSkuPrice(args.REGION, args.SKU, args.CPU, args.MEMORY_MB) + if err != nil { + return errors.Wrap(err, "GetSkuPrice") + } + fmt.Printf("result: %s\n", jsonutils.Marshal(result).PrettyString()) + return nil + }) + +} diff --git a/pkg/multicloud/google/shell/snapshot.go b/pkg/multicloud/google/shell/snapshot.go index a4181ec4bb..6973f6d6a2 100644 --- a/pkg/multicloud/google/shell/snapshot.go +++ b/pkg/multicloud/google/shell/snapshot.go @@ -34,10 +34,10 @@ func init() { return nil }) - type SnapshotShowOptions struct { + type SnapshotIdOptions struct { ID string } - shellutils.R(&SnapshotShowOptions{}, "snapshot-show", "Show snapshot", func(cli *google.SRegion, args *SnapshotShowOptions) error { + shellutils.R(&SnapshotIdOptions{}, "snapshot-show", "Show snapshot", func(cli *google.SRegion, args *SnapshotIdOptions) error { snapshot, err := cli.GetSnapshot(args.ID) if err != nil { return err @@ -46,4 +46,23 @@ func init() { return nil }) + shellutils.R(&SnapshotIdOptions{}, "snapshot-delete", "Delete snapshot", func(cli *google.SRegion, args *SnapshotIdOptions) error { + return cli.Delete(args.ID) + }) + + type SnapshotCreateOptions struct { + NAME string + Desc string + DISK string + } + + shellutils.R(&SnapshotCreateOptions{}, "snapshot-create", "Create snapshot", func(cli *google.SRegion, args *SnapshotCreateOptions) error { + snapshot, err := cli.CreateSnapshot(args.DISK, args.NAME, args.Desc) + if err != nil { + return err + } + printObject(snapshot) + return nil + }) + } diff --git a/pkg/multicloud/google/sku.go b/pkg/multicloud/google/sku.go new file mode 100644 index 0000000000..d638df9e82 --- /dev/null +++ b/pkg/multicloud/google/sku.go @@ -0,0 +1,352 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google + +import ( + "fmt" + "strings" + "time" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" +) + +type SkuBillingCatetory struct { + ServiceDisplayName string + ResourceFamily string + ResourceGroup string + UsageType string +} + +type SkuPricingInfo struct { + Summary string + PricingExpression SPricingExpression + currencyConversionRate int + EffectiveTime time.Time +} + +type SPricingExpression struct { + UsageUnit string + UsageUnitDescription string + BaseUnit string + BaseUnitDescription string + BaseUnitConversionFactor string + DisplayQuantity int + TieredRates []STieredRate +} + +type STieredRate struct { + StartUsageAmount int + UnitPrice SUnitPrice +} + +type SUnitPrice struct { + CurrencyCode string + Units string + Nanos int +} + +type SSkuBilling struct { + Name string + SkuId string + Description string + Category SkuBillingCatetory + ServiceRegions []string + PricingInfo []SkuPricingInfo + ServiceProviderName string +} + +func (region *SRegion) ListSkuBilling(pageSize int, pageToken string) ([]SSkuBilling, error) { + skus := []SSkuBilling{} + params := map[string]string{} + err := region.BillingList("services/6F81-5844-456A/skus", params, pageSize, pageToken, &skus) + if err != nil { + return nil, err + } + return skus, nil +} + +type SRateInfo struct { + // region: europe-north1 + // family: Compute, Storage, Network + // resource: CPU, Ram, Gpu, N1Standard + // category: custome, predefine, optimized + // startUsageAmount: + // map[region]map[family]map[resource]map[category]map[startUsageAmount][money] + + Info map[string]map[string]map[string]map[string]map[string]float64 +} + +func (region *SRegion) GetSkuRateInfo(skus []SSkuBilling) SRateInfo { + result := SRateInfo{ + Info: map[string]map[string]map[string]map[string]map[string]float64{}, + } + for _, sku := range skus { + if sku.ServiceProviderName == "Google" && + sku.Category.ServiceDisplayName == "Compute Engine" && + utils.IsInStringArray(sku.Category.ResourceFamily, []string{"Compute", "Storage"}) && + sku.Category.UsageType == "OnDemand" { + for _, region := range sku.ServiceRegions { + if _, ok := result.Info[region]; !ok { + result.Info[region] = map[string]map[string]map[string]map[string]float64{} + } + if _, ok := result.Info[region][sku.Category.ResourceFamily]; !ok { + result.Info[region][sku.Category.ResourceFamily] = map[string]map[string]map[string]float64{} + } + if sku.Category.ResourceGroup == "N1Standard" { + if strings.Index(sku.Description, "Core") > 0 { + sku.Category.ResourceGroup = "CPU" + } else if strings.Index(sku.Description, "Ram") > 0 { + sku.Category.ResourceGroup = "RAM" + } + } + if !utils.IsInStringArray(sku.Category.ResourceGroup, []string{"F1Micro", "G1Small", "CPU", "RAM", "PDStandard", "SSD", "LocalSSD", "f1-micro", "g1-small"}) { + continue + } + if utils.IsInStringArray(sku.Category.ResourceGroup, []string{"PDStandard", "SSD"}) && strings.Index(sku.Description, "Regional") >= 0 { + continue + } + convers := map[string]string{ + "PDStandard": api.STORAGE_GOOGLE_PD_STANDARD, + "SSD": api.STORAGE_GOOGLE_PD_SSD, + "LocalSSD": api.STORAGE_GOOGLE_LOCAL_SSD, + "F1Micro": "f1-micro", + "G1Small": "g1-small", + } + if group, ok := convers[sku.Category.ResourceGroup]; ok { + sku.Category.ResourceGroup = group + } + + if _, ok := result.Info[region][sku.Category.ResourceFamily][sku.Category.ResourceGroup]; !ok { + result.Info[region][sku.Category.ResourceFamily][sku.Category.ResourceGroup] = map[string]map[string]float64{} + } + description := strings.ToLower(sku.Description) + if strings.Contains("sole", description) { //单租户 + continue + } + category := "" + keys := []string{"memory optimized", "memory-optimized", "compute optimized", "n1 predefined", "n2 instance", "n2 custom extended", "custom extended", "n2 custom", "custom instance"} + categories := map[string]string{ + "memory optimized": "ultramem", + "memory-optimized": "memory-optimized", + "compute optimized": "compute-optimized", + + "n1 predefined": "n1-predefined", + "n2 instance": "n2-instance", + + "n2 custom extended": "n2-custom-extended", + "custom extended": "custom-extended", + + "n2 custom": "n2-custom", //cpu ram + "custom instance": "custom-instance", //cpu + } + for _, key := range keys { + _category := categories[key] + if strings.Contains(description, key) { + category = _category + break + } + } + if utils.IsInStringArray(sku.Category.ResourceGroup, []string{api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_LOCAL_SSD, "f1-micro", "g1-small"}) { + category = sku.Category.ResourceGroup + } + if len(category) == 0 { + continue + } + if _, ok := result.Info[region][sku.Category.ResourceFamily][sku.Category.ResourceGroup][category]; !ok { + result.Info[region][sku.Category.ResourceFamily][sku.Category.ResourceGroup][category] = map[string]float64{} + } + for _, priceInfo := range sku.PricingInfo { + for _, price := range priceInfo.PricingExpression.TieredRates { + result.Info[region][sku.Category.ResourceFamily][sku.Category.ResourceGroup][category][fmt.Sprintf("%d", price.StartUsageAmount)] = float64(price.UnitPrice.Nanos) / 1000000000 + } + } + } + } + } + return result +} + +func (rate *SRateInfo) GetDiscount(sku string) []float64 { + if strings.Index(sku, "custom") < 0 && strings.HasPrefix(sku, "c") || strings.Index(sku, "n2") >= 0 { + return []float64{0, 0.15, 0.25, 0.4} + } + return []float64{0, 0.2, 0.4, 0.6} +} + +func (rate *SRateInfo) GetSkuType(sku string) (string, error) { + if strings.Index(sku, "custom") >= 0 { + cpuType := "custom-instance" + if strings.HasPrefix(sku, "n2") { + cpuType = "n2-custom" + } + return cpuType, nil + } + if strings.HasPrefix(sku, "n1") { + return "n1-predefined", nil + } + if strings.HasPrefix(sku, "n2") { + return "n2-instance", nil + } + if strings.HasPrefix(sku, "c") { + return "compute-optimized", nil + } + if strings.HasPrefix(sku, "m") { + return "memory-optimized", nil + } + return "", fmt.Errorf("failed to found sku %s type", sku) +} + +func (rate *SRateInfo) GetCpuPrice(regionId, cpuType string) (float64, error) { + computePrice, err := rate.GetComputePrice(regionId) + if err != nil { + return 0, errors.Wrap(err, "GetComputePrice") + } + + _cpuPrice, ok := computePrice["CPU"] + if !ok { + return 0, fmt.Errorf("failed to found region %s compute cpu price info", regionId) + } + + cpuPrice, ok := _cpuPrice[cpuType] + if !ok { + return 0, fmt.Errorf("failed to found region %s compute %s cpu price info", regionId, cpuType) + } + return cpuPrice["0"], nil +} + +func (rate *SRateInfo) GetExtendMemoryGb(sku string, cpu int, memoryMb int) float64 { + maxMemoryGb := 0.0 + if strings.Index(sku, "custom") >= 0 { + maxMemoryGb = float64(cpu) * 6.5 + if strings.Index(sku, "n2") >= 0 { + maxMemoryGb = float64(cpu) * 8 + } + } + if float64(memoryMb)/1024 > maxMemoryGb { + return float64(memoryMb)/1024 - maxMemoryGb + } + return 0.0 +} + +func (rate *SRateInfo) GetMemoryPrice(regionId string, memoryType string) (float64, error) { + computePrice, err := rate.GetComputePrice(regionId) + if err != nil { + return 0, errors.Wrap(err, "GetComputePrice") + } + + _memoryPrice, ok := computePrice["RAM"] + if !ok { + return 0, fmt.Errorf("failed to found region %s compute memory price info", regionId) + } + + memoryPrice, ok := _memoryPrice[memoryType] + if !ok { + return 0, fmt.Errorf("failed to found region %s compute %s memory price info", regionId, memoryType) + } + return memoryPrice["0"], nil +} + +func (rate *SRateInfo) GetComputePrice(regionId string) (map[string]map[string]map[string]float64, error) { + regionPrice, ok := rate.Info[regionId] + if !ok { + return nil, fmt.Errorf("failed to found region %s price info", regionId) + } + computePrice, ok := regionPrice["Compute"] + if !ok { + return nil, fmt.Errorf("failed to found region %s compute price info", regionId) + } + return computePrice, nil +} + +func (rate *SRateInfo) GetSharedSkuPrice(regionId string, sku string) (float64, error) { + computePrice, err := rate.GetComputePrice(regionId) + if err != nil { + return 0.0, errors.Wrap(err, "GetComputePrice") + } + if _sharedSku, ok := computePrice[sku]; ok { + if sharedSku, ok := _sharedSku[sku]; ok { + return sharedSku["0"], nil + } + } + return 0.0, fmt.Errorf("sku is not shared sku") +} + +func (rate *SRateInfo) GetSkuPrice(regionId string, sku string, cpu, memoryMb int) (struct { + Hour float64 + Month float64 + Year float64 +}, error) { + result := struct { + Hour float64 + Month float64 + Year float64 + }{} + + discount := rate.GetDiscount(sku) + + price, err := rate.GetSharedSkuPrice(regionId, sku) + if err != nil { + skuType, err := rate.GetSkuType(sku) + if err != nil { + return result, errors.Wrap(err, "GetSkuType") + } + + cpuPrice, err := rate.GetCpuPrice(regionId, skuType) + if err != nil { + return result, errors.Wrap(err, "price.GetCpuPrice") + } + + price += float64(cpu) * cpuPrice + log.Debugf("cpu price: %f", cpuPrice) + + extendMemoryGb := rate.GetExtendMemoryGb(sku, cpu, memoryMb) + memoryMb = int(float64(memoryMb) - 1024*extendMemoryGb) + + memoryPrice, err := rate.GetMemoryPrice(regionId, skuType) + if err != nil { + return result, errors.Wrap(err, "GetMemoryPrice") + } + + price += memoryPrice * float64(memoryMb/1024) + log.Debugf("ramPrice: %f", memoryPrice) + + if extendMemoryGb > 0 { + memoryType := "custom-extended" + if strings.HasPrefix(sku, "n2") { + memoryType = "n2-custom-extended" + } + extendPrice, err := rate.GetMemoryPrice(regionId, memoryType) + if err != nil { + return result, errors.Wrap(err, "GetMemoryPrice.Extend") + } + price += extendPrice * float64(extendMemoryGb) + log.Debugf("extendPrice: %f", extendPrice) + } + } + + log.Debugf("totalPrice: %f", price) + + result.Month = 182.5 * price + result.Month += 182.5 * (1 - discount[1]) * price + result.Month += 182.5 * (1 - discount[2]) * price + result.Month += 172.49999999999997 * (1 - discount[3]) * price + result.Hour = result.Month / 30 / 24 + result.Year = result.Month * 12 + return result, nil +} diff --git a/pkg/multicloud/google/snapshot.go b/pkg/multicloud/google/snapshot.go index ada48a21fa..b89f61aa46 100644 --- a/pkg/multicloud/google/snapshot.go +++ b/pkg/multicloud/google/snapshot.go @@ -21,7 +21,6 @@ import ( "yunion.io/x/jsonutils" api "yunion.io/x/onecloud/pkg/apis/compute" - "yunion.io/x/onecloud/pkg/cloudprovider" ) type SSnapshot struct { @@ -106,7 +105,7 @@ func (snapshot *SSnapshot) GetDiskType() string { } func (snapshot *SSnapshot) Delete() error { - return cloudprovider.ErrNotImplemented + return snapshot.region.Delete(snapshot.SelfLink) } func (snapshot *SSnapshot) GetProjectId() string { diff --git a/pkg/multicloud/google/storage.go b/pkg/multicloud/google/storage.go index e56d72ba34..29c27f2464 100644 --- a/pkg/multicloud/google/storage.go +++ b/pkg/multicloud/google/storage.go @@ -120,11 +120,21 @@ func (storage *SStorage) GetEnabled() bool { } func (storage *SStorage) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) { - return nil, cloudprovider.ErrNotFound + disk, err := storage.zone.region.GetDisk(id) + if err != nil { + return nil, err + } + disk.storage = storage + return disk, nil } func (storage *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { - return nil, cloudprovider.ErrNotImplemented + disk, err := storage.zone.region.CreateDisk(name, sizeGb, storage.zone.Name, storage.Name, "", desc) + if err != nil { + return nil, err + } + disk.storage = storage + return disk, nil } func (storage *SStorage) GetMountPoint() string { diff --git a/pkg/multicloud/google/storagecache.go b/pkg/multicloud/google/storagecache.go index dc329b392d..30dce73fa4 100644 --- a/pkg/multicloud/google/storagecache.go +++ b/pkg/multicloud/google/storagecache.go @@ -16,13 +16,22 @@ package google import ( "context" - - "github.com/aliyun/aliyun-oss-go-sdk/oss" + "fmt" + "strings" + "unicode" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/image/options" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/qemuimg" ) type SStoragecache struct { @@ -85,21 +94,111 @@ func (cache *SStoragecache) GetPath() string { } func (cache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool) (string, error) { - return "", cloudprovider.ErrNotImplemented + if len(image.ExternalId) > 0 { + _image, err := cache.region.GetImage(image.ExternalId) + if err != nil { + log.Errorf("GetImage error: %v", err) + } else { + status := _image.GetStatus() + log.Debugf("UploadImage: Image external ID %s exists, status %s", image.ExternalId, status) + if status == api.CACHED_IMAGE_STATUS_READY { + return image.ExternalId, nil + } + err = cache.region.Delete(image.ExternalId) + if err != nil { + log.Errorf("failed to delete %s image %s", status, image.ExternalId) + } + } + } else { + log.Debugf("UploadImage: no external ID") + } + + return cache.uploadImage(ctx, userCred, image, isForce) +} + +func (region *SRegion) checkAndCreateBucket(bucketName string) (*SBucket, error) { + bucket, err := region.GetBucket(bucketName) + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + bucket, err = region.CreateBucket(bucketName, "") + if err != nil { + return nil, errors.Wrapf(err, "region.CreateBucket(%s)", bucketName) + } + } else { + return nil, errors.Wrapf(err, "region.StorageGet(%s)", bucketName) + } + } + return bucket, nil +} + +func (cache *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool) (string, error) { + s := auth.GetAdminSession(ctx, options.Options.Region, "") + + meta, reader, _, err := modules.Images.Download(s, image.ImageId, string(qemuimg.QCOW2), false) + if err != nil { + return "", err + } + log.Infof("meta data %s", meta) + + info := struct { + Id string + Name string + Size int64 + Description string + }{} + meta.Unmarshal(&info) + + bucketName := fmt.Sprintf("imagecache-%s", info.Id) + bucket, err := cache.region.checkAndCreateBucket(bucketName) + if err != nil { + return "", errors.Wrapf(err, "checkAndCreateBucket(%s)", bucketName) + } + + defer cache.region.DeleteBucket(bucket.Name) + + err = cache.region.PutObject(bucketName, info.Name, reader, "", info.Size, cloudprovider.ACLPublicRead) + if err != nil { + return "", errors.Wrap(err, "region.PutObject") + } + + images, err := cache.region.GetImages(cache.region.GetProjectId(), 0, "") + if err != nil { + return "", errors.Wrap(err, "region.GetImages") + } + imageNames := []string{} + for _, image := range images { + imageNames = append(imageNames, image.Name) + } + + imageName := "img-" + for _, s := range strings.ToLower(info.Name) { + if unicode.IsDigit(s) || unicode.IsLetter(s) || s == '-' { + imageName = fmt.Sprintf("%s%c", imageName, s) + } else { + imageName = fmt.Sprintf("%s-", imageName) + } + } + + baseName := imageName + for i := 0; i < 30; i++ { + if !utils.IsInStringArray(imageName, imageNames) { + break + } + imageName = fmt.Sprintf("%s-%d", baseName, i) + } + + _image, err := cache.region.CreateImage(imageName, info.Description, bucketName, info.Name) + if err != nil { + return "", errors.Wrap(err, "region.CreateImage") + } + + return _image.GetGlobalId(), nil } func (cache *SStoragecache) CreateIImage(snapshoutId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { return nil, cloudprovider.ErrNotImplemented } -func (cache *SRegion) CheckBucket(bucketName string) (*oss.Bucket, error) { - return nil, cloudprovider.ErrNotImplemented -} - -func (cache *SRegion) CreateImage(snapshoutId, imageName, imageDesc string) (string, error) { - return "", cloudprovider.ErrNotImplemented -} - func (cache *SStoragecache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/multicloud/google/vpc.go b/pkg/multicloud/google/vpc.go index 2e3fcc51e9..6f7e528bb6 100644 --- a/pkg/multicloud/google/vpc.go +++ b/pkg/multicloud/google/vpc.go @@ -53,7 +53,7 @@ func (vpc *SVpc) GetStatus() string { } func (vpc *SVpc) Delete() error { - return cloudprovider.ErrNotSupported + return vpc.region.Delete(vpc.globalnetwork.SelfLink) } func (vpc *SVpc) GetCidrBlock() string { @@ -82,8 +82,8 @@ func (vpc *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, erro return nil, errors.Wrap(err, "GetFirewalls") } isecgroups := []cloudprovider.ICloudSecurityGroup{} - tags := []string{} allInstance := false + tags := []string{} for _, firewall := range firewalls { if len(firewall.TargetServiceAccounts) > 0 { secgroup := &SSecurityGroup{vpc: vpc, ServiceAccount: firewall.TargetServiceAccounts[0]} diff --git a/pkg/multicloud/google/wire.go b/pkg/multicloud/google/wire.go index 21e73fbcbe..b9972d597b 100644 --- a/pkg/multicloud/google/wire.go +++ b/pkg/multicloud/google/wire.go @@ -39,7 +39,12 @@ func (wire *SWire) GetName() string { } func (wire *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - return nil, cloudprovider.ErrNotImplemented + network, err := wire.vpc.region.CreateNetwork(name, wire.vpc.globalnetwork.SelfLink, cidr, desc) + if err != nil { + return nil, err + } + network.wire = wire + return network, nil } func (wire *SWire) GetIVpc() cloudprovider.ICloudVpc { diff --git a/pkg/multicloud/huawei/instance.go b/pkg/multicloud/huawei/instance.go index 5967b5953e..b07ac33e42 100644 --- a/pkg/multicloud/huawei/instance.go +++ b/pkg/multicloud/huawei/instance.go @@ -519,25 +519,25 @@ func (self *SInstance) UpdateUserData(userData string) error { // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876971.html 更换系统盘操作系统 // 不支持调整系统盘大小 // todo: 支持注入user_data -func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { var err error var jobId string publicKeyName := "" - if len(publicKey) > 0 { - publicKeyName, err = self.host.zone.region.syncKeypair(publicKey) + if len(desc.PublicKey) > 0 { + publicKeyName, err = self.host.zone.region.syncKeypair(desc.PublicKey) if err != nil { return "", err } } - if self.Metadata.MeteringImageID == imageId { - jobId, err = self.host.zone.region.RebuildRoot(ctx, self.UserID, self.GetId(), passwd, publicKeyName, publicKey, self.OSEXTSRVATTRUserData) + if self.Metadata.MeteringImageID == desc.ImageId { + jobId, err = self.host.zone.region.RebuildRoot(ctx, self.UserID, self.GetId(), desc.Password, publicKeyName, desc.PublicKey, self.OSEXTSRVATTRUserData) if err != nil { return "", err } } else { - jobId, err = self.host.zone.region.ChangeRoot(ctx, self.UserID, self.GetId(), imageId, passwd, publicKeyName, publicKey, self.OSEXTSRVATTRUserData) + jobId, err = self.host.zone.region.ChangeRoot(ctx, self.UserID, self.GetId(), desc.ImageId, desc.Password, publicKeyName, desc.PublicKey, self.OSEXTSRVATTRUserData) if err != nil { return "", err } diff --git a/pkg/multicloud/instance_base.go b/pkg/multicloud/instance_base.go index 36296498fb..a0351eaf2b 100644 --- a/pkg/multicloud/instance_base.go +++ b/pkg/multicloud/instance_base.go @@ -14,6 +14,8 @@ package multicloud +import "yunion.io/x/onecloud/pkg/cloudprovider" + type SInstanceBase struct { SResourceBase } @@ -21,3 +23,7 @@ type SInstanceBase struct { func (instance *SInstanceBase) GetIHostId() string { return "" } + +func (instance *SInstanceBase) GetSerialOutput(port int) (string, error) { + return "", cloudprovider.ErrNotImplemented +} diff --git a/pkg/multicloud/openstack/instance.go b/pkg/multicloud/openstack/instance.go index b38e887407..79fe3cb3fa 100644 --- a/pkg/multicloud/openstack/instance.go +++ b/pkg/multicloud/openstack/instance.go @@ -442,8 +442,8 @@ func (instance *SInstance) DeployVM(ctx context.Context, name string, username s return instance.host.zone.region.DeployVM(instance.ID, name, password, publicKey, deleteKeypair, description) } -func (instance *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { - return instance.ID, instance.host.zone.region.ReplaceSystemDisk(instance.ID, imageId, passwd, publicKey, sysSizeGB) +func (instance *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { + return instance.ID, instance.host.zone.region.ReplaceSystemDisk(instance.ID, desc.ImageId, desc.Password, desc.PublicKey, desc.SysSizeGB) } func (instance *SInstance) ChangeConfig(ctx context.Context, config *cloudprovider.SManagedVMChangeConfig) error { diff --git a/pkg/multicloud/qcloud/instance.go b/pkg/multicloud/qcloud/instance.go index 70936aa19c..871084bf53 100644 --- a/pkg/multicloud/qcloud/instance.go +++ b/pkg/multicloud/qcloud/instance.go @@ -435,16 +435,16 @@ func (self *SInstance) DeployVM(ctx context.Context, name string, username strin return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, deleteKeypair, description) } -func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { keypair := "" - if len(publicKey) > 0 { + if len(desc.PublicKey) > 0 { var err error - keypair, err = self.host.zone.region.syncKeypair(publicKey) + keypair, err = self.host.zone.region.syncKeypair(desc.PublicKey) if err != nil { return "", err } } - err := self.host.zone.region.ReplaceSystemDisk(self.InstanceId, imageId, passwd, keypair, sysSizeGB) + err := self.host.zone.region.ReplaceSystemDisk(self.InstanceId, desc.ImageId, desc.Password, keypair, desc.SysSizeGB) if err != nil { return "", err } diff --git a/pkg/multicloud/ucloud/instance.go b/pkg/multicloud/ucloud/instance.go index c945574f19..ce46b864b4 100644 --- a/pkg/multicloud/ucloud/instance.go +++ b/pkg/multicloud/ucloud/instance.go @@ -431,8 +431,8 @@ func (self *SInstance) UpdateUserData(userData string) error { // todo:// 3.将原系统重装为不同类型的系统时(Linux->Windows)，不可选择保留数据盘； // 4.重装不同版本的系统时(CentOS6->CentOS7)，若选择保留数据盘，请注意数据盘的文件系统格式； // 5.若主机CPU低于2核，不可重装为Windows系统。 -func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { - if len(publicKey) > 0 { +func (self *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { + if len(desc.PublicKey) > 0 { return "", fmt.Errorf("DeployVM not support assign ssh keypair") } @@ -454,7 +454,7 @@ func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd s } } - err := self.host.zone.region.RebuildRoot(self.GetId(), imageId, passwd) + err := self.host.zone.region.RebuildRoot(self.GetId(), desc.ImageId, desc.Password) if err != nil { return "", err } diff --git a/pkg/multicloud/zstack/instance.go b/pkg/multicloud/zstack/instance.go index 98fd8ae39d..62038461f8 100644 --- a/pkg/multicloud/zstack/instance.go +++ b/pkg/multicloud/zstack/instance.go @@ -28,6 +28,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/billing" ) @@ -35,6 +36,7 @@ type SInstanceCdrome struct { } type SInstance struct { + multicloud.SInstanceBase host *SHost ZStackBasic @@ -378,8 +380,8 @@ func (instance *SInstance) DeployVM(ctx context.Context, name string, username s return nil } -func (instance *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { - return instance.host.zone.region.RebuildRoot(instance.UUID, imageId, sysSizeGB) +func (instance *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { + return instance.host.zone.region.RebuildRoot(instance.UUID, desc.ImageId, desc.SysSizeGB) } func (region *SRegion) RebuildRoot(instanceId, imageId string, sysSizeGB int) (string, error) { diff --git a/pkg/util/cloudinit/cloudconfig.go b/pkg/util/cloudinit/cloudconfig.go index f9c129572e..288776f5d4 100644 --- a/pkg/util/cloudinit/cloudconfig.go +++ b/pkg/util/cloudinit/cloudconfig.go @@ -55,7 +55,7 @@ type SWriteFile struct { type SUser struct { Name string - Passwd string + PlainTextPasswd string HashedPasswd string LockPasswd bool SshAuthorizedKeys []string @@ -161,7 +161,7 @@ func (u *SUser) Password(passwd string) *SUser { if err != nil { log.Errorf("GeneratePassword error %s", err) } else { - u.Passwd = hash + u.PlainTextPasswd = passwd u.HashedPasswd = hash } u.LockPasswd = false @@ -169,12 +169,24 @@ func (u *SUser) Password(passwd string) *SUser { return u } +func (u *SUser) PowerShellScripts() []string { + shells := []string{} + shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, u.Name)) + shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, u.Name)) + if len(u.PlainTextPasswd) > 0 { + shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, u.Name, u.PlainTextPasswd)) + } + // enable需要再设置密码之后，否则会出现Enable-LocalUser : Unable to update the password. The value provided for the new password does not meet the length, complexity, or history requirements of the domain + shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, u.Name)) + return shells +} + func (u *SUser) ShellScripts() []string { shells := []string{} shells = append(shells, fmt.Sprintf("useradd -m %s || true", u.Name)) - if len(u.Passwd) > 0 { - shells = append(shells, fmt.Sprintf("usermod -p '%s' %s", u.Passwd, u.Name)) + if len(u.HashedPasswd) > 0 { + shells = append(shells, fmt.Sprintf("usermod -p '%s' %s", u.HashedPasswd, u.Name)) } home := "/" + u.Name @@ -216,6 +228,11 @@ func (conf *SCloudConfig) UserDataScript() string { } shells = append(shells, conf.Runcmd...) + // 允许密码及root登录(谷歌云镜像默认会禁止root及密码登录) + shells = append(shells, `sed -i "s/.*PermitRootLogin.*/PermitRootLogin yes/g" /etc/ssh/sshd_config`) + shells = append(shells, `sed -i 's/.*PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config`) + shells = append(shells, `systemctl restart sshd`) + for _, pkg := range conf.Packages { shells = append(shells, "which yum &>/dev/null && yum install -y "+pkg) shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+pkg) @@ -226,6 +243,16 @@ func (conf *SCloudConfig) UserDataScript() string { return CLOUD_SHELL_HEADER + strings.Join(shells, "\n") } +func (conf *SCloudConfig) UserDataPowerShell() string { + shells := []string{} + for _, u := range conf.Users { + shells = append(shells, u.PowerShellScripts()...) + } + shells = append(shells, conf.Runcmd...) + + return strings.Join(shells, "\n") +} + func (conf *SCloudConfig) UserDataBase64() string { data := conf.UserData() return base64.StdEncoding.EncodeToString([]byte(data)) @@ -281,8 +308,8 @@ func (conf *SCloudConfig) MergeUser(u SUser) { for i := 0; i < len(conf.Users); i += 1 { if u.Name == conf.Users[i].Name { // replace conf user password with input - if len(u.Passwd) > 0 { - conf.Users[i].Passwd = u.Passwd + if len(u.PlainTextPasswd) > 0 { + conf.Users[i].PlainTextPasswd = u.PlainTextPasswd conf.Users[i].HashedPasswd = u.HashedPasswd conf.Users[i].LockPasswd = u.LockPasswd }