From bb5beccf47ba342e0ec5caa816e9b7f019881b23 Mon Sep 17 00:00:00 2001 From: TangBin Date: Thu, 16 Aug 2018 14:19:01 +0800 Subject: [PATCH 01/45] add more resources interfaces --- pkg/cloudprovider/resources.go | 8 +++++++- pkg/util/aliyun/instance.go | 20 ++++++++++++++++++++ pkg/util/esxi/virtualmachine.go | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index f388d66feb..bee54d3cd9 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -153,7 +153,12 @@ type ICloudVM interface { StopVM(isForce bool) error DeleteVM() error + UpdateVM(name string) error + RebuildRoot(imageId string) error + DeployVM(resetPassword bool, keypair string, deleteKeypair bool) error + ChangeConfig(instanceId string,ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) + AttachDisk(diskId string) error } type ICloudNic interface { @@ -193,7 +198,8 @@ type ICloudDisk interface { GetCacheMode() string GetMountpoint() string Delete() error - Resize(int64) error + + Resize(newSize int64) error } type ICloudVpc interface { diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 80a2e9c267..e6dbbf5610 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -539,3 +539,23 @@ func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules [ } return nil } + +func (self *SInstance) UpdateVM(name string) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) RebuildRoot(imageId string) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) DeployVM(resetPassword bool, keypair string, deleteKeypair bool) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) ChangeConfig(instanceId string,ncpu int, vmem int) error { + return fmt.Errorf("not implement") +} + +func (self *SInstance) AttachDisk(diskId string) error { + return fmt.Errorf("not implement") +} \ No newline at end of file diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 2005bbc9bc..d4aed7891f 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -209,3 +209,23 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { ret.Add(jsonutils.NewString(url), "url") return ret, nil } + +func (self *SVirtualMachine) UpdateVM(name string) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) RebuildRoot(imageId string) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) DeployVM(resetPassword bool, keypair string, deleteKeypair bool) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) ChangeConfig(instanceId string,ncpu int, vmem int) error { + return fmt.Errorf("not implement") +} + +func (self *SVirtualMachine) AttachDisk(diskId string) error { + return fmt.Errorf("not implement") +} \ No newline at end of file From 8feaa3538bd0dc9c1482cab4944b208e2e4c0dd9 Mon Sep 17 00:00:00 2001 From: TangBin Date: Mon, 20 Aug 2018 11:02:37 +0800 Subject: [PATCH 02/45] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E9=85=8D=E7=BD=AE=E7=AD=89API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/db/db_dispatcher.go | 1 - pkg/cloudprovider/resources.go | 8 +- pkg/compute/guestdrivers/aliyun.go | 298 ++++++++++++++----- pkg/compute/models/guests.go | 48 ++- pkg/compute/tasks/guest_create_disk_task.go | 109 +++++++ pkg/compute/tasks/guest_deploy_task.go | 6 +- pkg/compute/tasks/guest_rebuild_root_task.go | 2 +- pkg/util/aliyun/disk.go | 23 +- pkg/util/aliyun/host.go | 4 +- pkg/util/aliyun/instance.go | 194 ++++++++++-- pkg/util/aliyun/keypair.go | 31 ++ pkg/util/aliyun/shell/disk.go | 11 + pkg/util/aliyun/shell/instance.go | 57 ++++ pkg/util/aliyun/storage.go | 2 +- pkg/util/esxi/datacenter.go | 1 + pkg/util/esxi/storage.go | 2 +- pkg/util/esxi/virtualmachine.go | 20 ++ 17 files changed, 683 insertions(+), 134 deletions(-) diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 8b4179f50d..76a3a18c93 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -980,7 +980,6 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre return nil, httperrors.NewGeneralError(err) } item.PreUpdate(ctx, userCred, query, dataDict) - diff, err := manager.TableSpec().Update(item, func() error { filterData := dataDict.CopyIncludes(updateFields(manager, userCred)...) err = filterData.Unmarshal(item) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index f388d66feb..ec124341c0 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -153,7 +153,12 @@ type ICloudVM interface { StopVM(isForce bool) error DeleteVM() error + UpdateVM(name string) error + RebuildRoot(imageId string) error + DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error + ChangeConfig(instanceId string,ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) + AttachDisk(diskId string) error } type ICloudNic interface { @@ -193,7 +198,8 @@ type ICloudDisk interface { GetCacheMode() string GetMountpoint() string Delete() error - Resize(int64) error + + Resize(newSize int64) error } type ICloudVpc interface { diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 9477c541f6..d7489da423 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "time" - "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/httperrors" @@ -103,7 +102,6 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo imageId := disk.GetTemplateId() scimg := models.StoragecachedimageManager.GetStoragecachedimage(cache.Id, imageId) config.ExternalImageId = scimg.ExternalId - img := scimg.GetCachedimage() config.OsDistribution, _ = img.Info.GetString("properties", "os_distribution") config.OsVersion, _ = img.Info.GetString("properties", "os_version") @@ -134,92 +132,134 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return err } - if action != "create" { - return fmt.Errorf("Action %s not supported", action) - } - ihost, err := host.GetIHost() if err != nil { return err } - desc := SAliyunVMCreateConfig{} - err = config.Unmarshal(&desc, "desc") - if err != nil { - return err - } - - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - passwd := seclib2.RandomPassword2(12) - - iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, - desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey) + if action == "create" { + desc := SAliyunVMCreateConfig{} + err = config.Unmarshal(&desc, "desc") if err != nil { - return nil, err - } - log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) - err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) - if err != nil { - return nil, err - } - log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId()) - - iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) - if err != nil { - log.Errorf("cannot find vm %s", err) - return nil, err + return err } - if len(guest.SecgrpId) > 0 { - if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { - log.Errorf("SyncSecurityGroup error: %v", err) - return nil, err - } - } + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + passwd := seclib2.RandomPassword2(12) - if onfinish == "none" { - err = iVM.StartVM() + iVM, err := ihost.CreateVM(desc.Name, desc.ExternalImageId, desc.SysDiskSize, desc.Cpu, desc.Memory, desc.ExternalNetworkId, + desc.IpAddr, desc.Description, passwd, desc.StorageType, desc.DataDisks, desc.PublicKey) if err != nil { return nil, err } - } - - encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) - if err != nil { - log.Errorf("encrypt password failed %s", err) - } - - data := jsonutils.NewDict() - data.Add(jsonutils.NewString(iVM.GetOSType()), "os") - data.Add(jsonutils.NewString("root"), "account") - data.Add(jsonutils.NewString(encpasswd), "key") - - if len(desc.OsDistribution) > 0 { - data.Add(jsonutils.NewString(desc.OsDistribution), "distro") - } - if len(desc.OsVersion) > 0 { - data.Add(jsonutils.NewString(desc.OsVersion), "version") - } - - idisks, err := iVM.GetIDisks() - - if err != nil { - log.Errorf("GetiDisks error %s", err) - } else { - diskInfo := make([]SDiskInfo, len(idisks)) - for i := 0; i < len(idisks); i += 1 { - dinfo := SDiskInfo{} - dinfo.Uuid = idisks[i].GetGlobalId() - dinfo.Size = idisks[i].GetDiskSizeMB() - diskInfo[i] = dinfo + log.Debugf("VMcreated %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return nil, err } - data.Add(jsonutils.Marshal(&diskInfo), "disks") + log.Debugf("VMcreated %s, and status is ready", iVM.GetGlobalId()) + + iVM, err = ihost.GetIVMById(iVM.GetGlobalId()) + if err != nil { + log.Errorf("cannot find vm %s", err) + return nil, err + } + + if len(guest.SecgrpId) > 0 { + if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { + log.Errorf("SyncSecurityGroup error: %v", err) + return nil, err + } + } + + if onfinish == "none" { + err = iVM.StartVM() + if err != nil { + return nil, err + } + } + + encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) + if err != nil { + log.Errorf("encrypt password failed %s", err) + } + + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(iVM.GetOSType()), "os") + data.Add(jsonutils.NewString("root"), "account") + data.Add(jsonutils.NewString(encpasswd), "key") + + if len(desc.OsDistribution) > 0 { + data.Add(jsonutils.NewString(desc.OsDistribution), "distro") + } + if len(desc.OsVersion) > 0 { + data.Add(jsonutils.NewString(desc.OsVersion), "version") + } + + idisks, err := iVM.GetIDisks() + + if err != nil { + log.Errorf("GetiDisks error %s", err) + } else { + diskInfo := make([]SDiskInfo, len(idisks)) + for i := 0; i < len(idisks); i += 1 { + dinfo := SDiskInfo{} + dinfo.Uuid = idisks[i].GetGlobalId() + dinfo.Size = idisks[i].GetDiskSizeMB() + diskInfo[i] = dinfo + } + data.Add(jsonutils.Marshal(&diskInfo), "disks") + } + + data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + + return data, nil + }) + } else if action == "deploy" { + iVM, err := ihost.GetIVMById(guest.GetExternalId()) + if err != nil || iVM == nil { + log.Errorf("cannot find vm %s", err) + return fmt.Errorf("cannot find vm") } - data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + params := task.GetParams() + log.Debugf("Deploy VM params %s", params.String()) + var name string + if v, e := params.GetString("name"); e != nil { + name = v + } + var description string + if v, e := params.GetString("description"); e != nil { + description = v + } + resetPassword := jsonutils.QueryBoolean(params, "reset_password", false) + deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) + password, _ := params.GetString("password") + if resetPassword && len(password) == 0{ + password = seclib2.RandomPassword2(12) + } + + publicKey := "" + if k, e := config.GetString("public_key"); e != nil { + publicKey = k + } + + taskman.LocalTaskRun(task, func () (jsonutils.JSONObject, error) { + encpasswd, err := utils.EncryptAESBase64(guest.Id, password) + if err != nil { + log.Errorf("encrypt password failed %s", err) + } + + data := jsonutils.NewDict() + data.Add(jsonutils.NewString("root"), "account") // 用户名 + data.Add(jsonutils.NewString(encpasswd), "key") // 密码 + e := iVM.DeployVM(name, password, publicKey, resetPassword, deleteKeypair, description) + return data, e + }) + } else { + return fmt.Errorf("Action %s not supported", action) + } - return data, nil - }) return nil } @@ -277,3 +317,117 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue }) return nil } + +type SAliyunVMChangeConfig struct { + InstanceId string + Cpu int + Memory int +} + +func (self *SAliyunGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "AliyunGuestCreateDiskTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) + return nil +} + +func (self *SAliyunGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error { + config := SAliyunVMChangeConfig{} + config.InstanceId = guest.GetExternalId() + config.Cpu = int(vcpuCount) + config.Memory = int(vmemSize) + // taskman localTaskRun + ihost, err := guest.GetHost().GetIHost() + if err != nil { + return err + } + + iVM, err := ihost.GetIVMById(config.InstanceId) + if err != nil { + return err + } + + err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) + // todo: wait status ready & check disk + if err != nil { + return err + } + log.Debugf("VMchangeConfig %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*300) + if err != nil { + return err + } + log.Debugf("VMchangeConfig %s, and status is ready", iVM.GetGlobalId()) + return nil +} + +func (self *SAliyunGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { + ihost, e := host.GetIHost() + if e != nil { + return nil, e + } + + ivm, e := ihost.GetIVMById(guest.GetExternalId()) + if e != nil { + return nil, e + } + + err := ivm.StartVM() + if err != nil { + return nil, e + } + + result := jsonutils.NewDict() + result.Add(jsonutils.NewBool(true), "is_running") + return result, e +} + +func (self *SAliyunGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + ihost, e := guest.GetHost().GetIHost() + if e != nil { + return e + } + + externalId := guest.GetExternalId() + if len(externalId) <= 0 { + return fmt.Errorf("external id not found") + } + + disks := guest.GetDisks() + if len(disks) <= 0 { + return fmt.Errorf("guest has no disk") + } + + imageId := guest.CategorizeDisks().Root.TemplateId + cacheId := disks[0].GetDisk().GetStorage().GetStoragecache().Id + externalImageId := models.StoragecachedimageManager.GetStoragecachedimage(cacheId, imageId).ExternalId + if len(externalImageId) <= 0 { + return fmt.Errorf("external image (%s) id is not found", imageId) + } + + iVM, err := ihost.GetIVMById(externalId) + if err != nil { + return err + } + + err = iVM.RebuildRoot(externalImageId) + if err != nil { + return err + } + + log.Debugf("VMrebuildRoot %s, wait status ready ...", iVM.GetGlobalId()) + err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*1800) + if err != nil { + return err + } + log.Debugf("VMrebuildRoot %s, and status is ready", iVM.GetGlobalId()) + + task.ScheduleRun(nil) + return nil +} \ No newline at end of file diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index c5dec34277..09cb942918 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -472,9 +472,11 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To return nil, err } - // if data.Contains("name") { - // return nil, httperrors.NewInputParameterError("cannot update server name") - // } + if data.Contains("name") { + if name, _ := data.GetString("name"); len(name) < 2 { + return nil, httperrors.NewInputParameterError("name is to short") + } + } /* if self.GetHypervisor() == HYPERVISOR_BAREMETAL { return nil, httperrors.NewInputParameterError("Cannot modify memory for baremetal") } @@ -1507,7 +1509,11 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr if !ok { return nil, fmt.Errorf("Parse query body error") } + + // 变更密码/密钥时需要Restart才能生效。更新普通字段不需要Restart + doRestart := false if kwargs.Contains("__delete_keypair__") || kwargs.Contains("keypair") { + doRestart = true var kpId string if !jsonutils.QueryBoolean(kwargs, "__delete_keypair__", false) { keypair, _ := kwargs.GetString("keypair") @@ -1529,8 +1535,9 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr kwargs.Set("reset_password", jsonutils.JSONTrue) } } + if utils.IsInStringArray(self.Status, []string{VM_RUNNING, VM_READY, VM_ADMIN}) { - if self.Status == VM_RUNNING { + if doRestart && self.Status == VM_RUNNING { kwargs.Set("restart", jsonutils.JSONTrue) } err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "") @@ -1874,12 +1881,10 @@ func (self *SGuest) createDiskOnHost(ctx context.Context, userCred mcclient.Toke if storage == nil { return nil, fmt.Errorf("No storage to create disk") } - disk, err := self.createDiskOnStorage(ctx, userCred, storage, diskConfig, pendingUsage) if err != nil { return nil, err } - err = self.attach2Disk(disk, userCred, diskConfig.Driver, diskConfig.Cache, diskConfig.Mountpoint) return disk, err } @@ -2412,16 +2417,27 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T diskIdx += 1 } - for storageId, needSize := range diskSizes { - iStorage, err := StorageManager.FetchById(storageId) - if err != nil { - return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) - } - storage := iStorage.(*SStorage) - if storage.GetFreeCapacity() < needSize { - return nil, httperrors.NewInsufficientResourceError("Not enough free space") - } + provider, e := self.GetHost().GetDriver() + if e != nil { + log.Errorf("Get Provider Error: %s", e) + return nil, httperrors.NewInsufficientResourceError("Provider Not Found") } + + if !provider.IsPublicCloud() { + for storageId, needSize := range diskSizes { + iStorage, err := StorageManager.FetchById(storageId) + if err != nil { + return nil, httperrors.NewBadRequestError("Fetch storage error: %s", err) + } + storage := iStorage.(*SStorage) + if storage.GetFreeCapacity() < needSize { + return nil, httperrors.NewInsufficientResourceError("Not enough free space") + } + } + } else { + log.Debugf("Skip storage free capacity validating for public cloud: %s", provider.GetName()) + } + if newDisks.Length() > 0 { confs.Add(newDisks, "create") } @@ -3293,4 +3309,4 @@ func (manager *SGuestManager) CleanPendingDeleteServers(ctx context.Context, use for i := 0; i < len(guests); i += 1 { guests[i].StartDeleteGuestTask(ctx, userCred, "", false, true) } -} +} \ No newline at end of file diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index c7034bd735..bc734944cd 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/log" ) type GuestCreateDiskTask struct { @@ -118,7 +119,115 @@ func (self *KVMGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, ob self.SetStageComplete(ctx, nil) } +type AliyunGuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *AliyunGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStage("on_aliyun_disk_prepared", nil) + self.OnAliyunDiskPrepared(ctx, obj, data) +} + +func (self *AliyunGuestCreateDiskTask) OnAliyunDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + var diskIndex = 0 + var diskReady = true + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status == models.DISK_INIT { + snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + if err != nil { + snapInfo = "" + } + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + diskReady = false + break + } + diskIndex += 1 + } + diskIndex = 0 + guest := obj.(*models.SGuest) + + for { + diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) + if !diskReady || err != nil { + break + } + iDisk, err := models.DiskManager.FetchById(diskId) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if iDisk == nil { + self.SetStageFailed(ctx, "Disk not found") + return + } + disk := iDisk.(*models.SDisk) + if disk.Status != models.DISK_READY { + diskReady = false + break + } + + ihost, err := guest.GetHost().GetIHost() + if err != nil { + self.SetStageFailed(ctx, "Host not found") + return + } + + iVM, e := ihost.GetIVMById(guest.GetExternalId()) + if e != nil { + self.SetStageFailed(ctx, "Aliyun VM not found") + return + } + + err = iVM.AttachDisk(disk.GetExternalId()) + if err != nil { + log.Debugf("Attach Disk %s to guest fail: %s", diskId, err) + self.SetStageFailed(ctx, "Attach Disk to guest fail") + return + } + diskIndex += 1 + } + + if diskReady { + if guest.Status == models.VM_RUNNING { + self.SetStage("on_config_sync_complete", nil) + err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + } + } else { + self.SetStageComplete(ctx, nil) + } + } +} + +func (self *AliyunGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func (self *AliyunGuestCreateDiskTask) AttachAliyunDisks(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + func init() { taskman.RegisterTask(GuestCreateDiskTask{}) taskman.RegisterTask(KVMGuestCreateDiskTask{}) + taskman.RegisterTask(AliyunGuestCreateDiskTask{}) } diff --git a/pkg/compute/tasks/guest_deploy_task.go b/pkg/compute/tasks/guest_deploy_task.go index 47fd39b7c7..037a909a1c 100644 --- a/pkg/compute/tasks/guest_deploy_task.go +++ b/pkg/compute/tasks/guest_deploy_task.go @@ -54,14 +54,10 @@ func (self *GuestDeployTask) StartDeployGuestOnHost(ctx context.Context, guest * log.Errorf("request_deploy_guest_on_host %s", err) self.OnDeployGuestFail(ctx, guest, err) } else { - self.OnDeployGuestSucc(guest) + guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") } } -func (self *GuestDeployTask) OnDeployGuestSucc(guest *models.SGuest) { - guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") -} - func (self *GuestDeployTask) OnDeployGuestFail(ctx context.Context, guest *models.SGuest, err error) { guest.SetStatus(self.UserCred, models.VM_DEPLOY_FAILED, err.Error()) self.SetStageFailed(ctx, err.Error()) diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go index 03345b85ac..50de80c0e9 100644 --- a/pkg/compute/tasks/guest_rebuild_root_task.go +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -141,4 +141,4 @@ func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { self.SetStageComplete(ctx, nil) -} +} \ No newline at end of file diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index aeeca8096c..ca6f21d4f8 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -138,6 +138,12 @@ func (self *SDisk) Refresh() error { return jsonutils.Update(self, new) } +func (self *SDisk) ResizeDisk(newSize int64) error { + // newSize 单位为 GB. 范围在20 ～2000. 只能往大调。不能调小 + // https://help.aliyun.com/document_detail/25522.html?spm=a2c4g.11174283.6.897.aHwqkS + return self.storage.zone.region.resizeDisk(self.DiskId, newSize) +} + func (self *SDisk) GetDiskFormat() string { return "vhd" } @@ -185,7 +191,7 @@ func (self *SDisk) GetMountpoint() string { return "" } -func (self *SRegion) createDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) { +func (self *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string) (string, error) { params := make(map[string]string) params["ZoneId"] = zoneId params["DiskName"] = name @@ -223,11 +229,24 @@ func (self *SRegion) deleteDisk(diskId string) error { return err } +func (self *SRegion) DeleteDisk(diskId string) error { + params := make(map[string]string) + params["DiskId"] = diskId + + _, err := self.ecsRequest("DeleteDisk", params) + return err +} + func (self *SRegion) resizeDisk(diskId string, size int64) error { params := make(map[string]string) params["DiskId"] = diskId params["NewSize"] = fmt.Sprintf("%d", size) _, err := self.ecsRequest("ResizeDisk", params) - return err + if err != nil { + log.Errorf("ResizeDisk %s to %s GiB fail %s", diskId, size, err) + return err + } + + return nil } diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index 3ee4a40236..cf842879ad 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -157,7 +157,7 @@ func (self *SHost) GetManagerId() string { return self.zone.region.client.providerId } -func (self *SHost) getInstanceById(instanceId string) (*SInstance, error) { +func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) { inst, err := self.zone.region.GetInstance(instanceId) if err != nil { return nil, err @@ -171,7 +171,7 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, if err != nil { return nil, err } - vm, err := self.getInstanceById(vmId) + vm, err := self.GetInstanceById(vmId) if err != nil { return nil, err } diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 80a2e9c267..9bf836a180 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" "time" + "yunion.io/x/onecloud/pkg/util/seclib2" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -110,6 +111,7 @@ type SInstance struct { InternetMaxBandwidthIn int InternetMaxBandwidthOut int IoOptimized bool + KeyPairName string Memory int NetworkInterfaces SNetworkInterfaces OSName string @@ -322,6 +324,81 @@ func (self *SInstance) GetHypervisor() string { return models.HYPERVISOR_ALIYUN } +func (self *SInstance) StartVM() error { + err := self.host.zone.region.StartVM(self.InstanceId) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes +} + +func (self *SInstance) StopVM(isForce bool) error { + err := self.host.zone.region.StopVM(self.InstanceId, isForce) + if err != nil { + return err + } + return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues +} + +func (self *SInstance) DeleteVM() error { + err := self.host.zone.region.DeleteVM(self.InstanceId) + if err != nil { + return err + } + return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes +} + +func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { + url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId) + if err != nil { + return nil, err + } + passwd := seclib.RandomPassword(6) + err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd) + if err != nil { + return nil, err + } + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(url), "url") + ret.Add(jsonutils.NewString(passwd), "password") + ret.Add(jsonutils.NewString("aliyun"), "protocol") + ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") + return ret, nil +} + +func (self *SInstance) UpdateVM(name string) error { + return self.host.zone.region.UpdateVM(self.InstanceId, name) +} + +func (self *SInstance) DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error { + var keypairName string + if len(publicKey) > 0 { + key, e := self.host.lookUpAliyunKeypair(publicKey) + if e != nil { + key, e = self.host.importAliyunKeypair(publicKey) + if e != nil { + return e + } + } + + keypairName = key + } + + return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, resetPassword, deleteKeypair, description) +} + +func (self *SInstance) RebuildRoot(imageId string) error { + return self.host.zone.region.ReplaceSystemDisk(self.InstanceId, imageId) +} + +func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { + return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil) +} + +func (self *SInstance) AttachDisk(diskId string) error { + return self.host.zone.region.AttachDisk(self.InstanceId, diskId) +} + func (self *SRegion) GetInstance(instanceId string) (*SInstance, error) { instances, _, err := self.GetInstances("", []string{instanceId}, 0, 1) if err != nil { @@ -472,46 +549,99 @@ func (self *SRegion) DeleteVM(instanceId string) error { // } } -func (self *SInstance) StartVM() error { - err := self.host.zone.region.StartVM(self.InstanceId) +func (self *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, resetPassword bool, deleteKeypair bool, description string) error { + instance, err := self.GetInstance(instanceId) if err != nil { return err } - return cloudprovider.WaitStatus(self, models.VM_RUNNING, 5*time.Second, 180*time.Second) // 3minutes + + // 修改密钥时直接返回 + if deleteKeypair { + return self.DetachKeyPair(instanceId, instance.KeyPairName) + } + + if len(keypairName) > 0 { + return self.AttachKeypair(instanceId, keypairName) + } + + params := make(map[string]string) + if resetPassword { + params["Password"] = seclib2.RandomPassword2(12) + } + // 指定密码的情况下，使用指定的密码 + if len(password) > 0 { + params["Password"] = password + } + + if len(name) > 0 && instance.InstanceName != name { + params["InstanceName"] = name + params["HostName"] = name + } + + if len(description) > 0 && instance.Description != description { + params["Description"] = description + } + + if len(params) > 0 { + log.Debugf("DeployVM with params %s", params) + return self.modifyInstanceAttribute(instanceId, params) + } else { + return nil + } } -func (self *SInstance) StopVM(isForce bool) error { - err := self.host.zone.region.StopVM(self.InstanceId, isForce) +func (self *SRegion) UpdateVM(instanceId string, hostname string) error { + /* + api: ModifyInstanceAttribute + https://help.aliyun.com/document_detail/25503.html?spm=a2c4g.11186623.4.1.DrgpjW + */ + params := make(map[string]string) + params["HostName"] = hostname + return self.modifyInstanceAttribute(instanceId, params) +} + +func (self *SRegion) modifyInstanceAttribute(instanceId string, params map[string]string) error { + return self.instanceOperation(instanceId, "ModifyInstanceAttribute", params) +} + +func (self *SRegion) ReplaceSystemDisk(instanceId string, image string) error { + params := make(map[string]string) + params["ImageId"] = image + return self.instanceOperation(instanceId, "ReplaceSystemDisk", params) +} + +func (self *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, vmem int, disks []*SDisk) error { + // todo: support change disk config? + params := make(map[string]string) + instanceTypes, e := self.GetMatchInstanceTypes(ncpu, vmem, 0, zoneId) + if e != nil { + return e + } + + for _, instancetype := range instanceTypes { + params["InstanceType"] = instancetype.InstanceTypeId + params["ClientToken"] = utils.GenRequestId(20) + if err := self.instanceOperation(instanceId, "ModifyInstanceSpec", params); err != nil { + log.Errorf("Failed for %s: %s", instancetype.InstanceTypeId, err) + } else { + return nil + } + } + + return fmt.Errorf("Failed to change vm config, specification not supported") +} + +func (self *SRegion) AttachDisk(instanceId string, diskId string) error { + params := make(map[string]string) + params["InstanceId"] = instanceId + params["DiskId"] = diskId + _, err := self.ecsRequest("AttachDisk", params) if err != nil { + log.Errorf("AttachDisk %s to %s fail %s", diskId, instanceId, err) return err } - return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues -} -func (self *SInstance) DeleteVM() error { - err := self.host.zone.region.DeleteVM(self.InstanceId) - if err != nil { - return err - } - return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes -} - -func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { - url, err := self.host.zone.region.GetInstanceVNCUrl(self.InstanceId) - if err != nil { - return nil, err - } - passwd := seclib.RandomPassword(6) - err = self.host.zone.region.ModifyInstanceVNCUrlPassword(self.InstanceId, passwd) - if err != nil { - return nil, err - } - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(url), "url") - ret.Add(jsonutils.NewString(passwd), "password") - ret.Add(jsonutils.NewString("aliyun"), "protocol") - ret.Add(jsonutils.NewString(self.InstanceId), "instance_id") - return ret, nil + return nil } func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules []secrules.SecurityRule) error { @@ -538,4 +668,4 @@ func (self *SInstance) SyncSecurityGroup(secgroupId string, name string, rules [ self.SecurityGroupIds.SecurityGroupId = []string{secgrpId} } return nil -} +} \ No newline at end of file diff --git a/pkg/util/aliyun/keypair.go b/pkg/util/aliyun/keypair.go index c8da45b6a0..e3b2a504a8 100644 --- a/pkg/util/aliyun/keypair.go +++ b/pkg/util/aliyun/keypair.go @@ -1,6 +1,7 @@ package aliyun import ( + "encoding/json" "fmt" "yunion.io/x/log" ) @@ -62,3 +63,33 @@ func (self *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error } return &keypair, nil } + +func (self *SRegion) AttachKeypair(instanceId string, name string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["KeyPairName"] = name + instances, _ := json.Marshal(&[...]string{instanceId}) + params["InstanceIds"] = string(instances) + _, err := self.ecsRequest("AttachKeyPair", params) + if err != nil { + log.Errorf("AttachKeyPair fail %s", err) + return err + } + + return nil +} + +func (self *SRegion) DetachKeyPair(instanceId string, name string) error { + params := make(map[string]string) + params["RegionId"] = self.RegionId + params["KeyPairName"] = name + instances, _ := json.Marshal(&[...]string{instanceId}) + params["InstanceIds"] = string(instances) + _, err := self.ecsRequest("DetachKeyPair", params) + if err != nil { + log.Errorf("DetachKeyPair fail %s", err) + return err + } + + return nil +} diff --git a/pkg/util/aliyun/shell/disk.go b/pkg/util/aliyun/shell/disk.go index 1196bb5739..19cc4c7693 100644 --- a/pkg/util/aliyun/shell/disk.go +++ b/pkg/util/aliyun/shell/disk.go @@ -21,4 +21,15 @@ func init() { printList(disks, total, args.Offset, args.Limit, []string{}) return nil }) + + type DiskDeleteOptions struct { + Instance string `help:"Instance ID"` + } + shellutils.R(&DiskDeleteOptions{}, "disk-delete", "List disks", func(cli *aliyun.SRegion, args *DiskDeleteOptions) error { + e := cli.DeleteDisk(args.Instance) + if e != nil { + return e + } + return nil + }) } diff --git a/pkg/util/aliyun/shell/instance.go b/pkg/util/aliyun/shell/instance.go index 495e2949cf..4a741d0b66 100644 --- a/pkg/util/aliyun/shell/instance.go +++ b/pkg/util/aliyun/shell/instance.go @@ -82,6 +82,63 @@ func init() { return nil }) + /* + server-change-config 更改系统配置 + server-reset + */ + type InstanceDeployOptions struct { + ID string `help:"instance ID"` + Name string `help:"new instance name"` + Hostname string `help:"new hostname"` + Keypair string `help:"Keypair Name"` + DeleteKeypair bool `help:"Remove SSH keypair"` + Password string `help:"new password"` + ResetPassword bool `help:"Force reset password"` + Description string `help:"new instances description"` + } + + shellutils.R(&InstanceDeployOptions{}, "instance-deploy", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceDeployOptions) error { + err := cli.DeployVM(args.ID, args.Name, args.Password, args.Keypair, args.ResetPassword, args.DeleteKeypair, args.Description) + if err != nil { + return err + } + return nil + }) + + type InstanceRebuildRootOptions struct { + ID string `help:"instance ID"` + Image string `help:"Image ID"` + } + + shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Reinstall virtual server system image", func(cli *aliyun.SRegion, args *InstanceRebuildRootOptions) error { + err := cli.ReplaceSystemDisk(args.ID, args.Image) + if err != nil { + return err + } + return nil + }) + + type InstanceChangeConfigOptions struct { + ID string `help:"instance ID"` + Ncpu int `help:"number of CPU"` + Vmem int `help:"MiB of memory"` + Disk []int `help:"Data disk sizes int GB"` + } + + shellutils.R(&InstanceChangeConfigOptions{}, "instance-change-config", "Deploy keypair/password to a stopped virtual server", func(cli *aliyun.SRegion, args *InstanceChangeConfigOptions) error { + instance, e := cli.GetInstance(args.ID) + if e != nil { + return e + } + + // todo : add create disks + err := cli.ChangeVMConfig(instance.ZoneId, args.ID, args.Ncpu, args.Vmem, nil) + if err != nil { + return err + } + return nil + }) + type InstanceUpdatePasswordOptions struct { ID string `help:"Instance ID"` PASSWD string `help:"new password"` diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index b54700b1e6..8f66d8b68b 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -100,7 +100,7 @@ func (self *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache { } func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { - diskId, err := self.zone.region.createDisk(self.zone.ZoneId, self.storageType, name, sizeGb, desc) + diskId, err := self.zone.region.CreateDisk(self.zone.ZoneId, self.storageType, name, sizeGb, desc) if err != nil { log.Errorf("createDisk fail %s", err) return nil, err diff --git a/pkg/util/esxi/datacenter.go b/pkg/util/esxi/datacenter.go index 7da30e1dfb..6c9d1c6d0d 100644 --- a/pkg/util/esxi/datacenter.go +++ b/pkg/util/esxi/datacenter.go @@ -76,3 +76,4 @@ func (dc *SDatacenter) GetIStorages() ([]cloudprovider.ICloudStorage, error) { } return dc.istorages, nil } + diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 90846ed5a9..dc154261d2 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -89,4 +89,4 @@ func (self *SDatastore) GetManagerId() string { func (self *SDatastore) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented -} +} \ No newline at end of file diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 2005bbc9bc..75ccccc448 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -65,6 +65,22 @@ func (self *SVirtualMachine) IsEmulated() bool { return false } +func (self *SVirtualMachine) DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) RebuildRoot(imageId string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) UpdateVM(name string) error { + return cloudprovider.ErrNotImplemented +} + +func (self *SVirtualMachine) AttachDisk(diskId string) error { + return cloudprovider.ErrNotImplemented +} + func (self *SVirtualMachine) getUuid() string { return self.getVirtualMachine().Summary.Config.Uuid } @@ -209,3 +225,7 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { ret.Add(jsonutils.NewString(url), "url") return ret, nil } + +func (dc *SVirtualMachine) ChangeConfig(instanceId string,ncpu int, vmem int) error { + return cloudprovider.ErrNotImplemented +} \ No newline at end of file From d33757fbbb19ab2c09e82363a12b62094972ee1c Mon Sep 17 00:00:00 2001 From: TangBin Date: Mon, 20 Aug 2018 15:10:46 +0800 Subject: [PATCH 03/45] =?UTF-8?q?=E5=A2=9E=E5=8A=A0CPU=E3=80=81=E5=86=85?= =?UTF-8?q?=E5=AD=98=E5=8F=98=E6=9B=B4=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/aliyun.go | 11 ++++++----- pkg/util/aliyun/instance.go | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index d7489da423..7baad16137 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -342,7 +342,6 @@ func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest config.InstanceId = guest.GetExternalId() config.Cpu = int(vcpuCount) config.Memory = int(vmemSize) - // taskman localTaskRun ihost, err := guest.GetHost().GetIHost() if err != nil { return err @@ -353,11 +352,13 @@ func (self *SAliyunGuestDriver) RequestChangeVmConfig(ctx context.Context, guest return err } - err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) - // todo: wait status ready & check disk - if err != nil { - return err + if int(guest.VcpuCount) != config.Cpu || guest.VmemSize != config.Memory { + err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) + if err != nil { + return err + } } + log.Debugf("VMchangeConfig %s, wait status ready ...", iVM.GetGlobalId()) err = cloudprovider.WaitStatus(iVM, models.VM_READY, time.Second*5, time.Second*300) if err != nil { diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 9bf836a180..e74189be1c 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -583,7 +583,6 @@ func (self *SRegion) DeployVM(instanceId string, name string, password string, k } if len(params) > 0 { - log.Debugf("DeployVM with params %s", params) return self.modifyInstanceAttribute(instanceId, params) } else { return nil From a85f3d71768e78c65cb91cfce4b6802537aec37e Mon Sep 17 00:00:00 2001 From: TangBin Date: Thu, 23 Aug 2018 09:43:09 +0800 Subject: [PATCH 04/45] =?UTF-8?q?=E9=94=99=E8=AF=AF=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E6=96=B0=E5=A2=9Edata=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/httperrors/errors.go | 193 +++++++++++++------------------- pkg/httperrors/httperrors.go | 8 +- pkg/util/httputils/httputils.go | 6 + 3 files changed, 89 insertions(+), 118 deletions(-) diff --git a/pkg/httperrors/errors.go b/pkg/httperrors/errors.go index d57bf8ff79..c447191304 100644 --- a/pkg/httperrors/errors.go +++ b/pkg/httperrors/errors.go @@ -6,37 +6,48 @@ import ( "yunion.io/x/onecloud/pkg/util/httputils" ) -func NewJsonClientError(code int, title string, msg string) *httputils.JSONClientError { - err := httputils.JSONClientError{Code: code, Class: title, Details: msg} +func NewJsonClientError(code int, title string, msg string, error httputils.Error) *httputils.JSONClientError { + err := httputils.JSONClientError{Code: code, Class: title, Details: msg, Data: error} return &err } -func NewBadGatewayError(msg string, params ...interface{}) *httputils.JSONClientError { +func errorMessage(msg string, params ...interface{}) (string, httputils.Error) { + fileds := make([]string, len(params)) + for i, v := range params { + fileds[i] = fmt.Sprint(v) + } + + error := httputils.Error{Id: msg, Fields: fileds} if len(params) > 0 { msg = fmt.Sprintf(msg, params...) } - return NewJsonClientError(502, "BadGateway", msg) + + return msg, error +} + +func NewBadGatewayError(msg string, params ...interface{}) *httputils.JSONClientError { + msg, err := errorMessage(msg, params) + return NewJsonClientError(502, "BadGateway", msg, err) } func NewNotImplementedError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(501, "NotImplemented", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(501, "NotImplemented", msg, err) } func NewInternalServerError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(500, "InternalServerError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(500, "InternalServerError", msg, err) } func NewResourceNotReadyError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(500, "ResourceNotReadyError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(500, "ResourceNotReadyError", msg, err) +} + +func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONClientError { + msg, err := errorMessage(msg, params) + return NewJsonClientError(500, "NewOutOfResourceError", msg, err) } func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONClientError { @@ -47,178 +58,128 @@ func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONCli } func NewServerStatusError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(400, "ServerStatusError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(400, "ServerStatusError", msg, err) } func NewPaymentError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(402, "PaymentError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(402, "PaymentError", msg, err) } func NewImageNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(404, "ImageNotFoundError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(404, "ImageNotFoundError", msg, err) } func NewResourceNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(404, "ResourceNotFoundError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(404, "ResourceNotFoundError", msg, err) } func NewSpecNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(404, "SpecNotFoundError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(404, "SpecNotFoundError", msg, err) } func NewActionNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(404, "ActionNotFoundError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(404, "ActionNotFoundError", msg, err) } func NewTenantNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(404, "TenantNotFoundError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(404, "TenantNotFoundError", msg, err) } func NewUserNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(404, "UserNotFoundError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(404, "UserNotFoundError", msg, err) } func NewInvalidStatusError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(400, "InvalidStatusError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(400, "InvalidStatusError", msg, err) } func NewInputParameterError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(400, "InputParameterError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(400, "InputParameterError", msg, err) } func NewInsufficientResourceError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(400, "InsufficientResourceError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(400, "InsufficientResourceError", msg, err) } func NewOutOfQuotaError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(400, "OutOfQuotaError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(400, "OutOfQuotaError", msg, err) } func NewNotSufficientPrivilegeError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(403, "NotSufficientPrivilegeError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(403, "NotSufficientPrivilegeError", msg, err) } func NewUnsupportOperationError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(406, "UnsupportOperationError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(406, "UnsupportOperationError", msg, err) } func NewNotEmptyError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(406, "NotEmptyError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(406, "NotEmptyError", msg, err) } func NewBadRequestError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(400, "BadRequestError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(400, "BadRequestError", msg, err) } func NewUnauthorizedError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(401, "UnauthorizedError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(401, "UnauthorizedError", msg, err) } func NewInvalidCredentialError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(401, "InvalidCredentialError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(401, "InvalidCredentialError", msg, err) } func NewForbiddenError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(403, "ForbiddenError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(403, "ForbiddenError", msg, err) } func NewNotFoundError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(404, "NotFoundError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(404, "NotFoundError", msg, err) } func NewNotAcceptableError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(406, "NotAcceptableError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(406, "NotAcceptableError", msg, err) } func NewDuplicateNameError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(409, "DuplicateNameError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(409, "DuplicateNameError", msg, err) } func NewConflictError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(409, "ConflictError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(409, "ConflictError", msg, err) } func NewResourceBusyError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(409, "ResourceBusyError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(409, "ResourceBusyError", msg, err) } func NewRequireLicenseError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(402, "RequireLicenseError", msg) + msg, err := errorMessage(msg, params) + return NewJsonClientError(402, "RequireLicenseError", msg, err) } func NewGeneralError(err error) *httputils.JSONClientError { diff --git a/pkg/httperrors/httperrors.go b/pkg/httperrors/httperrors.go index a696ab0394..6fc0d8c741 100644 --- a/pkg/httperrors/httperrors.go +++ b/pkg/httperrors/httperrors.go @@ -8,18 +8,22 @@ import ( "yunion.io/x/onecloud/pkg/util/httputils" ) -func HTTPError(w http.ResponseWriter, msg string, statusCode int, class string) { +func HTTPError(w http.ResponseWriter, msg string, statusCode int, class string, error httputils.Error) { w.WriteHeader(statusCode) w.Header().Set("Content-Type", "application/json") body := jsonutils.NewDict() body.Add(jsonutils.NewInt(int64(statusCode)), "code") body.Add(jsonutils.NewString(msg), "details") body.Add(jsonutils.NewString(class), "class") + err := jsonutils.NewDict() + err.Add(jsonutils.NewString(error.Id), "id") + err.Add(jsonutils.NewStringArray(error.Fields), "fields") + body.Add(err, "data") w.Write([]byte(body.String())) } func JsonClientError(w http.ResponseWriter, e *httputils.JSONClientError) { - HTTPError(w, e.Details, e.Code, e.Class) + HTTPError(w, e.Details, e.Code, e.Class, e.Data) } func GeneralServerError(w http.ResponseWriter, e error) { diff --git a/pkg/util/httputils/httputils.go b/pkg/util/httputils/httputils.go index 603f99987a..efd7cfecd5 100644 --- a/pkg/util/httputils/httputils.go +++ b/pkg/util/httputils/httputils.go @@ -32,10 +32,16 @@ var ( cyan = color.New(color.FgHiCyan, color.Bold).PrintlnFunc() ) +type Error struct { + Id string + Fields []string +} + type JSONClientError struct { Code int Class string Details string + Data Error } func (e *JSONClientError) Error() string { From 57c1766568863a3124032ef79ec7196ccba3c6ab Mon Sep 17 00:00:00 2001 From: TangBin Date: Thu, 23 Aug 2018 18:15:06 +0800 Subject: [PATCH 05/45] errors add variable params --- pkg/httperrors/errors.go | 7 ------ pkg/httperrors/httperrors.go | 48 ++++++++++++++++++------------------ 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/pkg/httperrors/errors.go b/pkg/httperrors/errors.go index c447191304..0daa2cc0b4 100644 --- a/pkg/httperrors/errors.go +++ b/pkg/httperrors/errors.go @@ -50,13 +50,6 @@ func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONCli return NewJsonClientError(500, "NewOutOfResourceError", msg, err) } -func NewOutOfResourceError(msg string, params ...interface{}) *httputils.JSONClientError { - if len(params) > 0 { - msg = fmt.Sprintf(msg, params...) - } - return NewJsonClientError(500, "NewOutOfResourceError", msg) -} - func NewServerStatusError(msg string, params ...interface{}) *httputils.JSONClientError { msg, err := errorMessage(msg, params) return NewJsonClientError(400, "ServerStatusError", msg, err) diff --git a/pkg/httperrors/httperrors.go b/pkg/httperrors/httperrors.go index 6fc0d8c741..84dc0d3216 100644 --- a/pkg/httperrors/httperrors.go +++ b/pkg/httperrors/httperrors.go @@ -35,52 +35,52 @@ func GeneralServerError(w http.ResponseWriter, e error) { } } -func BadRequestError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewBadRequestError(msg)) +func BadRequestError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewBadRequestError(msg, params...)) } -func PaymentError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewPaymentError(msg)) +func PaymentError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewPaymentError(msg, params...)) } -func UnauthorizedError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewUnauthorizedError(msg)) +func UnauthorizedError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewUnauthorizedError(msg, params...)) } -func InvalidCredentialError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewInvalidCredentialError(msg)) +func InvalidCredentialError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewInvalidCredentialError(msg, params...)) } -func ForbiddenError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewForbiddenError(msg)) +func ForbiddenError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewForbiddenError(msg, params...)) } -func NotFoundError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewNotFoundError(msg)) +func NotFoundError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewNotFoundError(msg, params...)) } -func NotImplementedError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewNotImplementedError(msg)) +func NotImplementedError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewNotImplementedError(msg, params...)) } -func NotAcceptableError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewNotAcceptableError(msg)) +func NotAcceptableError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewNotAcceptableError(msg, params...)) } -func InvalidInputError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewInputParameterError(msg)) +func InvalidInputError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewInputParameterError(msg, params...)) } -func ConflictError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewConflictError(msg)) +func ConflictError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewConflictError(msg, params...)) } -func InternalServerError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewInternalServerError(msg)) +func InternalServerError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewInternalServerError(msg, params...)) } -func BadGatewayError(w http.ResponseWriter, msg string) { - JsonClientError(w, NewBadGatewayError(msg)) +func BadGatewayError(w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(w, NewBadGatewayError(msg, params...)) } func TenantNotFoundError(w http.ResponseWriter, msg string, params ...interface{}) { From b23562b019cc80c5d8ae1862f60c41429f612ad5 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 23 Aug 2018 20:39:10 +0800 Subject: [PATCH 06/45] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E9=98=BF=E9=87=8C=E4=BA=91VPC=E6=97=B6=E7=94=B1?= =?UTF-8?q?=E4=BA=8E=E6=9C=89securitygroup=E5=AF=BC=E8=87=B4=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/util/aliyun/securitygroup.go | 12 ++++++++++++ pkg/util/aliyun/vpc.go | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 85d9518b32..969bce79c8 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -189,3 +189,15 @@ func (self *SRegion) createDefaultSecurityGroup(vpcId string) (string, error) { } return secId, nil } + +func (self *SRegion) deleteSecurityGroup(secGrpId string) error { + params := make(map[string]string) + params["SecurityGroupId"] = secGrpId + + _, err := self.ecsRequest("DeleteSecurityGroup", params) + if err != nil { + log.Errorf("Delete security group fail %s", err) + return err + } + return nil +} diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index d61028a555..196aaddb42 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/log" ) const ( @@ -178,5 +179,17 @@ func (self *SVpc) GetManagerId() string { } func (self *SVpc) Delete() error { + err := self.fetchSecurityGroups() + if err != nil { + log.Errorf("fetchSecurityGroup for VPC delete fail %s", err) + return err + } + for i := 0; i < len(self.secgroups); i += 1 { + err := self.region.deleteSecurityGroup(self.secgroups[i].SecurityGroupId) + if err != nil { + log.Errorf("deleteSecurityGroup for VPC delete fail %s", err) + return err + } + } return self.region.DeleteVpc(self.VpcId) } From cc63b9ff9374e1b3fcd97e771c1c26fb0decfd86 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 14:38:56 +0800 Subject: [PATCH 07/45] =?UTF-8?q?=E6=9B=B4=E6=96=B0Makefile=EF=BC=8C?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0make=20dep=E5=91=BD=E4=BB=A4=EF=BC=8C?= =?UTF-8?q?=E5=8F=AA=E6=9B=B4=E6=96=B0vendor/yunion.io/x/=E4=B8=8B?= =?UTF-8?q?=E7=9A=84=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 228037d447..5984634fd0 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ REPO_PREFIX := yunion.io/x/onecloud VENDOR_PATH := $(REPO_PREFIX)/vendor VERSION_PKG := $(VENDOR_PATH)/yunion.io/x/pkg/util/version -ROOT_DIR := $(shell pwd) +ROOT_DIR := $(CURDIR) BUILD_DIR := $(ROOT_DIR)/_output BIN_DIR := $(BUILD_DIR)/bin BUILD_SCRIPT := $(ROOT_DIR)/build/build.sh @@ -95,5 +95,8 @@ fmt: find . -type f -name "*.go" -not -path "./_output/*" \ -not -path "./vendor/*" | xargs gofmt -s -w +dep: + cd $(ROOT_DIR) && dep ensure -v -update $(shell for p in $$(ls vendor/yunion.io/x/); do echo "yunion.io/x/$$p"; done | xargs) + %: @: From f539aa53d8400a77f65e269a9eb0c7eef4ebe180 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 14:39:40 +0800 Subject: [PATCH 08/45] Update vendor & Gopkg.lock --- Gopkg.lock | 16 ++++++++-------- vendor/yunion.io/x/sqlchemy/field_update.go | 2 +- vendor/yunion.io/x/sqlchemy/insert.go | 2 +- vendor/yunion.io/x/sqlchemy/update.go | 2 +- vendor/yunion.io/x/structarg/structarg.go | 3 +++ 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 2499f89c7f..aba3be928b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1026,11 +1026,11 @@ [[projects]] branch = "master" - digest = "1:5ede93047a3e04f6ede8fbc011f13ee94c397b727613fd9712cc99ebc47ff906" + digest = "1:d0257638bb52243f9fa293ef07544081259d54edd8e53f084e522b6c22e92637" name = "yunion.io/x/jsonutils" packages = ["."] pruneopts = "UT" - revision = "41e805b221e8fcd9435b706b4dd473df3dd8cff4" + revision = "38477c9cceb895816fe21507d73da30376d358b7" [[projects]] branch = "master" @@ -1045,7 +1045,7 @@ [[projects]] branch = "master" - digest = "1:13da776b435ec7873c6f9309191296d920a110145372d2a9e1a58ab9e8b34b0d" + digest = "1:47670c70d0326f624de21c8f29d0f73850062f0752f27e146308db62fbb540d3" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1080,23 +1080,23 @@ "utils", ] pruneopts = "UT" - revision = "56b426f0ca15288cc26442f5802c949f137ade60" + revision = "6c349ecbdb90d5f6ecc9b86596428176c1937b32" [[projects]] branch = "master" - digest = "1:3d3e1ecb41c63448df5d66eeafa5f6da720ca6e32543e89faa82b5bc4b9adebc" + digest = "1:e23019db2cda58480528738c86bc7abbff09d10488c2a51dd5d500f8686815f0" name = "yunion.io/x/sqlchemy" packages = ["."] pruneopts = "UT" - revision = "a74ef73e555a1ed2e19b1a47e0567814c289eaf9" + revision = "19a3c90524d3638f8e3580231396dda46088aebc" [[projects]] branch = "master" - digest = "1:6baf7b4ee14e156a9dd6c5c850bb8201af47665e3267e3cdee26edf2a90eaf73" + digest = "1:71e1b62648868a9083f9e4693145a0154a600591ca548aaf64cae32a10e956e5" name = "yunion.io/x/structarg" packages = ["."] pruneopts = "UT" - revision = "d5e5d87357b9bc2164215117763f6b15b2a3e75d" + revision = "4a5eb8e2cdfbf7f6511561a3c710c116b171f405" [solve-meta] analyzer-name = "dep" diff --git a/vendor/yunion.io/x/sqlchemy/field_update.go b/vendor/yunion.io/x/sqlchemy/field_update.go index 2087dbb6c0..b6b32203a5 100644 --- a/vendor/yunion.io/x/sqlchemy/field_update.go +++ b/vendor/yunion.io/x/sqlchemy/field_update.go @@ -102,7 +102,7 @@ func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{} buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = NOW()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) } buf.WriteString(" WHERE ") first = true diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index d9f10406c1..c7442588f0 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -36,7 +36,7 @@ func (t *STableSpec) insertSqlPrep(dataFields map[string]interface{}) (string, [ if ok && (dtc.IsCreatedAt || dtc.IsUpdatedAt) { createdAtFields = append(createdAtFields, k) names = append(names, fmt.Sprintf("`%s`", k)) - format = append(format, "NOW()") + format = append(format, "UTC_TIMESTAMP()") } else if ov != nil && !c.IsZero(ov) && !isAutoInc { v := c.ConvertFromValue(ov) values = append(values, v) diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index d99ef0b95e..e40e0f6eff 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -126,7 +126,7 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, er buf.WriteString(fmt.Sprintf(", `%s` = `%s` + 1", versionField, versionField)) } for _, updatedField := range updatedFields { - buf.WriteString(fmt.Sprintf(", `%s` = NOW()", updatedField)) + buf.WriteString(fmt.Sprintf(", `%s` = UTC_TIMESTAMP()", updatedField)) } buf.WriteString(" WHERE ") first = true diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index 9eb223f272..7f7f6c6414 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -305,6 +305,9 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) return fmt.Errorf("positional %s must not have default value", token) } } + if !positional && use_default && required { + return fmt.Errorf("non-positional argument with default value should not have required:true set") + } subcommand, err := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) if err != nil { subcommand = false From 038d1c731ba14390ec319b4c28605afef98f1b59 Mon Sep 17 00:00:00 2001 From: TangBin Date: Fri, 24 Aug 2018 15:08:13 +0800 Subject: [PATCH 09/45] bugfix --- pkg/mcclient/modules/mod_domains.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/mcclient/modules/mod_domains.go b/pkg/mcclient/modules/mod_domains.go index 84d57ffb34..e7be85cd5e 100644 --- a/pkg/mcclient/modules/mod_domains.go +++ b/pkg/mcclient/modules/mod_domains.go @@ -240,12 +240,7 @@ func (this *DomainManager) DoDomainConfigDelete(s *mcclient.ClientSession, param return ret, httperrors.NewResourceNotFoundError("找不到该认证域") } - driver, err := detail.GetString("driver") - if err != nil { - log.Errorf("got driver from domain detail error: %v", err) - return ret, httperrors.NewInternalServerError("服务器错误,获取认证协议失败,不允许删除") - } - + driver, _ := detail.GetString("driver") if driver != "ldap" { if result, err := UsersV3.List(s, params); err != nil { log.Errorf("user list got error: %v", err) From 58aecb256d43b765f69d6060236cb5268577827b Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 16:15:13 +0800 Subject: [PATCH 10/45] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E4=B8=BB=E6=9C=BA=E6=97=B6=E5=8F=AA=E9=80=89=E6=8B=A9?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E4=B8=BA=20available=20=E7=9A=84=E7=BD=91?= =?UTF-8?q?=E7=BB=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/wires.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 22fa6d9a6f..d522c53e58 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -347,6 +347,7 @@ func (self *SWire) getNetworks() ([]SNetwork, error) { func (self *SWire) getGatewayNetworkQuery() *sqlchemy.SQuery { q := self.getNetworkQuery() q = q.IsNotNull("guest_gateway").IsNotEmpty("guest_gateway") + q = q.Equals("status", NETWORK_STATUS_AVAILABLE) return q } From 931ee5cef64fe6407f8067739f449a62fa96ca7e Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 16:30:56 +0800 Subject: [PATCH 11/45] =?UTF-8?q?=E5=A2=9E=E5=8A=A0VM=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/util/aliyun/instance.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 292a231193..5d90dedd32 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -422,6 +422,7 @@ func (self *SInstance) waitStatus(target string, interval time.Duration, timeout func (self *SRegion) StartVM(instanceId string) error { status, _ := self.GetInstanceStatus(instanceId) if status != InstanceStatusStopped { + log.Errorf("StartVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus } return self.doStartVM(instanceId) @@ -434,6 +435,7 @@ func (self *SRegion) StartVM(instanceId string) error { func (self *SRegion) StopVM(instanceId string, isForce bool) error { status, _ := self.GetInstanceStatus(instanceId) if status != InstanceStatusRunning { + log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus } return self.doStopVM(instanceId, isForce) @@ -451,6 +453,7 @@ func (self *SRegion) DeleteVM(instanceId string) error { return err } } else if status != InstanceStatusStopped { + log.Errorf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus } return self.doDeleteVM(instanceId) From 32d56101dfe81f42f44d29e585422512c50f230f Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Fri, 24 Aug 2018 16:32:08 +0800 Subject: [PATCH 12/45] minor updates --- pkg/util/aliyun/instance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 5d90dedd32..12cc379e68 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -435,7 +435,7 @@ func (self *SRegion) StartVM(instanceId string) error { func (self *SRegion) StopVM(instanceId string, isForce bool) error { status, _ := self.GetInstanceStatus(instanceId) if status != InstanceStatusRunning { - log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusStopped) + log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusRunning) return cloudprovider.ErrInvalidStatus } return self.doStopVM(instanceId, isForce) From f23c0257cd613b25beac60b9bb6036b60d5c57f8 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 09:10:32 +0000 Subject: [PATCH 13/45] dnsrecords: do not log NotFound as error --- pkg/compute/models/dnsrecords.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/compute/models/dnsrecords.go b/pkg/compute/models/dnsrecords.go index a1177a3401..5da570532c 100644 --- a/pkg/compute/models/dnsrecords.go +++ b/pkg/compute/models/dnsrecords.go @@ -6,7 +6,6 @@ import ( "strings" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/util/regutils" "yunion.io/x/sqlchemy" @@ -205,7 +204,6 @@ func (man *SDnsRecordManager) QueryDns(projectId, name string) *SDnsRecord { err := q.First(rec) if err != nil { - log.Errorf("QueryDns %q fail: %s", name, err) return nil } return rec From 745f97e912eda8811b05892f81e637ce34a9cdb2 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 09:11:16 +0000 Subject: [PATCH 14/45] guestnetworks: GetGuestByAddress(): optimize by using a single join query --- pkg/compute/models/guestnetworks.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 3e8a3685ac..2f3defdf37 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -263,17 +263,17 @@ func (self *SGuestnetwork) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest { - gn := SGuestnetwork{} - gn.SetModelManager(GuestnetworkManager) - - err := manager.Query().Equals("ip_addr", address).First(&gn) - if err != nil { - log.Errorf("GetGuestByAddress query fail: %s", err) - return nil - } - guest, _ := GuestManager.FetchById(gn.GuestId) - if guest != nil { - return guest.(*SGuest) + networks := manager.Query().SubQuery() + guests := GuestManager.Query() + q := guests.Join(networks, sqlchemy.AND( + sqlchemy.Equals(networks.Field("ip_addr"), address), + sqlchemy.Equals(networks.Field("guest_id"), guests.Field("id")), + )) + guest := &SGuest{} + guest.SetModelManager(GuestManager) + err := q.First(guest) + if err == nil { + return guest } return nil } From 96ceaafd2ff27a544d11b3960a18849e250b2185 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 12:30:12 +0000 Subject: [PATCH 15/45] hostnetworks: GetHostNetworkByAddress(): fix logic invert --- pkg/compute/models/hostnetworks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/hostnetworks.go b/pkg/compute/models/hostnetworks.go index abf31b3ed6..09875cce28 100644 --- a/pkg/compute/models/hostnetworks.go +++ b/pkg/compute/models/hostnetworks.go @@ -92,7 +92,7 @@ func (man *SHostnetworkManager) QueryByAddress(addr string) *sqlchemy.SQuery { func (man *SHostnetworkManager) GetHostNetworkByAddress(addr string) *SHostnetwork { network := SHostnetwork{} err := man.QueryByAddress(addr).First(&network) - if err != nil { + if err == nil { return &network } return nil From 54ee08ef494a81c4876a178436a4c0d3ce17a3da Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 12:33:56 +0000 Subject: [PATCH 16/45] hostnetworks: GetHostByAddress(): use a single join --- pkg/compute/models/hostnetworks.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/compute/models/hostnetworks.go b/pkg/compute/models/hostnetworks.go index 09875cce28..88433fb90a 100644 --- a/pkg/compute/models/hostnetworks.go +++ b/pkg/compute/models/hostnetworks.go @@ -107,9 +107,17 @@ func (man *SHostnetworkManager) GetNetworkByAddress(addr string) *SNetwork { } func (man *SHostnetworkManager) GetHostByAddress(addr string) *SHost { - net := man.GetHostNetworkByAddress(addr) - if net == nil { - return nil + networks := man.Query().SubQuery() + hosts := HostManager.Query() + q := hosts.Join(networks, sqlchemy.AND( + sqlchemy.Equals(networks.Field("ip_addr"), addr), + sqlchemy.Equals(networks.Field("baremetal_id"), hosts.Field("id")), + )) + host := &SHost{} + host.SetModelManager(HostManager) + err := q.First(host) + if err == nil { + return host } - return net.GetHost() + return nil } From 7679056f550eb14d466e353c838f41fcd831d708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Fri, 24 Aug 2018 20:58:54 +0800 Subject: [PATCH 17/45] =?UTF-8?q?=E6=B7=BB=E5=8A=A0storage=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/storages.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 4ac8ea7587..7990188dd8 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -12,6 +12,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" + "yunion.io/x/pkg/util/sysutils" "yunion.io/x/sqlchemy" ) @@ -654,6 +655,23 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. q = q.Filter(sqlchemy.In(q.Field("zone_id"), sq.SubQuery())) } + if jsonutils.QueryBoolean(query, "share", false) { + q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + } + + if jsonutils.QueryBoolean(query, "local", false) { + q = q.Filter(sqlchemy.In(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + } + + if jsonutils.QueryBoolean(query, "usable", false) { + hostStorage := HoststorageManager.Query().SubQuery() + q = q.Join(hostStorage, sqlchemy.AND( + sqlchemy.Equals(hostStorage.Field("storage_id"), q.Field("id")), + sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE}), + sqlchemy.IsTrue(q.Field("enabled")), + )) + } + managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) if len(managerStr) > 0 { provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr) From 8df37525c033bdcc59c914d04c833c3d9fc8168d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Fri, 24 Aug 2018 20:31:20 +0800 Subject: [PATCH 18/45] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9B=A0=E8=BE=93?= =?UTF-8?q?=E5=85=A5disk=20name=E5=AF=BC=E8=87=B4=E6=89=BE=E4=B8=8D?= =?UTF-8?q?=E5=88=B0=E7=A3=81=E7=9B=98,=E5=8D=B8=E8=BD=BD=E7=A3=81?= =?UTF-8?q?=E7=9B=98=E5=A4=B1=E8=B4=A5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 8ecba69d47..fe551cf561 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2337,7 +2337,7 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok disk.SetStatus(userCred, DISK_DETACHING, "") } taskData := jsonutils.NewDict() - taskData.Add(jsonutils.NewString(diskId), "disk_id") + taskData.Add(jsonutils.NewString(disk.Id), "disk_id") taskData.Add(jsonutils.NewBool(keepDisk), "keep_disk") self.GetDriver().StartGuestDetachdiskTask(ctx, userCred, self, taskData, "") return nil, nil From 5de6220a745d025ab27c80734b478dd4ff35f567 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Fri, 24 Aug 2018 22:25:33 +0800 Subject: [PATCH 19/45] server shot desc --- pkg/compute/models/baremetalstatus.go | 22 +++++ pkg/compute/models/disks.go | 21 +++++ pkg/compute/models/guests.go | 125 +++++++++++++++++++++++++ pkg/compute/models/hosts.go | 83 ++++++++++++++++ pkg/compute/models/isolated_devices.go | 18 ++++ 5 files changed, 269 insertions(+) create mode 100644 pkg/compute/models/baremetalstatus.go diff --git a/pkg/compute/models/baremetalstatus.go b/pkg/compute/models/baremetalstatus.go new file mode 100644 index 0000000000..5c4561f14c --- /dev/null +++ b/pkg/compute/models/baremetalstatus.go @@ -0,0 +1,22 @@ +package models + +const ( + BAREMETAL_INIT = "init" + BAREMETAL_PREPARE = "prepare" + BAREMETAL_PREPARE_FAIL = "prepare_fail" + BAREMETAL_READY = "ready" + BAREMETAL_RUNNING = "running" + BAREMETAL_MAINTAINING = "maintaining" + BAREMETAL_START_MAINTAIN = "start_maintain" + BAREMETAL_DELETING = "deleting" + BAREMETAL_DELETE = "delete" + BAREMETAL_DELETE_FAIL = "delete_fail" + BAREMETAL_UNKNOWN = "unknown" + BAREMETAL_SYNCING_STATUS = "syncing_status" + BAREMETAL_SYNC = "sync" + BAREMETAL_SYNC_FAIL = "sync_fail" + BAREMETAL_START_CONVERT = "start_convert" + BAREMETAL_CONVERTING = "converting" + BAREMETAL_START_FAIL = "start_fail" + BAREMETAL_STOP_FAIL = "stop_fail" +) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index a2e827b7b9..94dd5972d1 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -701,6 +701,27 @@ func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, info s return image.Id, nil } +// def get_disk_spec_v2(conf): +// def _get_spec(storages): +// spec = {} +// for adapter, ss in group_by_adapter(storages).items(): +// if len(ss) == 0: +// continue +// spec[adapter] = get_disk_spec(ss) +// return spec + +// spec = {} +// for driver in DISK_DRIVERS: +// storages = [s for s in conf if s['driver'] == driver] +// if len(storages) != 0: +// spec[driver] = _get_spec(storages) +// return spec + +func GetDiskSpecV2(storageInfo jsonutils.JSONObject) *jsonutils.JSONDict { + // ToDo + return nil +} + func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { if len(diskConfig.ImageId) > 0 { self.TemplateId = diskConfig.ImageId diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 8ecba69d47..aef8301dc4 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3027,6 +3027,131 @@ func (manager *SGuestManager) FetchGuestById(guestId string) *SGuest { return guest.(*SGuest) } +func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { + if checkStatus { + if !utils.IsInStringArray(self.Status, []string{VM_SCHEDULE_FAILED}) { + return nil + } + } + spec := jsonutils.NewDict() + spec.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount))) + spec.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) + + // get disk spec + guestdisks := self.GetDisks() + diskSpecs := jsonutils.NewArray() + for _, guestdisk := range guestdisks { + disk := guestdisk.GetDisk() + diskSpec := jsonutils.NewDict() + diskSpec.Set("size", jsonutils.NewInt(int64(disk.DiskSize))) + s := disk.GetStorage() + diskSpec.Set("backend", jsonutils.NewString(s.StorageType)) + diskSpec.Set("medium_type", jsonutils.NewString(s.MediumType)) + diskSpecs.Add(diskSpec) + } + spec.Set("disk", diskSpecs) + + // get nic spec + guestnics := self.GetNetworks() + nicSpecs := jsonutils.NewArray() + for _, guestnic := range guestnics { + nicSpec := jsonutils.NewDict() + nicSpec.Set("bandwidth", jsonutils.NewInt(int64(guestnic.getBandwidth()))) + t := "int" + if guestnic.IsExit() { + t = "ext" + } + nicSpec.Set("type", jsonutils.NewString(t)) + nicSpecs.Add(nicSpec) + } + spec.Set("nic", nicSpecs) + + // get isolate device spec + guestgpus := self.GetIsolatedDevices() + gpuSpecs := jsonutils.NewArray() + for _, guestgpu := range guestgpus { + if strings.HasPrefix(guestgpu.DevType, "GPU") { + gs := guestgpu.GetSpec(false) + if gs != nil { + gpuSpecs.Add(gs) + } + } + } + spec.Set("gpu", gpuSpecs) + return spec +} + +func (self *SGuest) GetTemplateId() string { + guestdisks := self.GetDisks() + for _, guestdisk := range guestdisks { + disk := guestdisk.GetDisk() + if disk != nil { + templateId := disk.GetTemplateId() + if len(templateId) > 0 { + return templateId + } + } + } + return "" +} + +// def get_short_desc(self, user_cred=None): +// desc = super(Guests, self).get_short_desc() +// desc['mem'] = self.vmem_size +// desc['cpu'] = self.vcpu_count +// template_id = self.get_template_id() +// if template_id is not None: +// desc['template_id'] = template_id +// ext_bw, int_bw = self.get_bandwidth() +// if ext_bw > 0: +// desc['ext_bandwidth'] = ext_bw +// if int_bw > 0: +// desc['int_bandwidth'] = int_bw +// hypervisor = self.get_hypervisor() +// desc['hypervisor'] = hypervisor +// spec = self.get_spec(user_cred, status_check=False) +// if hypervisor == self.HYPERVISOR_BAREMETAL: +// host = self.get_host() +// if host is not None: +// host_spec = '/'.join(host.get_spec_ident( +// host.get_spec(user_cred, status_check=False))) +// spec['host_spec'] = host_spec +// if spec is not None: +// desc.update(spec) +// return desc + +func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { + desc := self.SStandaloneResourceBase.GetShortDesc() + desc.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) + desc.Set("cpu", jsonutils.NewInt(int64(self.VcpuCount))) + templateId := self.GetTemplateId() + if len(templateId) > 0 { + desc.Set("cpu", jsonutils.NewString(templateId)) + } + extBw := self.getBandwidth(true) + intBw := self.getBandwidth(false) + if extBw > 0 { + desc.Set("ext_bandwidth", jsonutils.NewInt(int64(extBw))) + } + if intBw > 0 { + desc.Set("int_bandwidth", jsonutils.NewInt(int64(intBw))) + } + desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) + spec := self.GetSpec(false) + if self.GetHypervisor() == HYPERVISOR_BAREMETAL { + host := self.GetHost() + if host != nil { + hostSpec := host.GetSpec(false) + hostSpecIdent := host.GetSpecIdent(hostSpec) + spec.Set("host_spec", jsonutils.NewString(strings.Join(hostSpecIdent, "/"))) + } + } + if spec != nil { + desc.Update(spec) + } + return desc +} + func (self *SGuest) saveOsType(osType string) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.OsType = osType diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 6b47c4e370..97ccecb6ef 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -506,6 +506,89 @@ func (self *SHost) ClearSchedDescCache() error { return HostManager.ClearSchedDescCache(self.Id) } +// def get_spec(self, user_cred, status_check=True, **kwargs): +// if status_check: +// if self.status in [baremetalstatus.INIT, baremetalstatus.PREPARE_FAIL, +// baremetalstatus.PREPARE] or \ +// self.get_baremetal_server() is not None: +// return None +// if self.mem_size is None or self.cpu_count is None: +// return None +// from clouds.baremetal import nictypes +// spec = self.get_hardware_specification() +// spec.pop('storage_info', None) +// net_info = [] +// for netif in self.get_netifs(): +// net_desc = netif.get_baremetal_json_desc() +// if net_desc.get('nic_type', None) != nictypes.NIC_TYPE_IPMI: +// net_info.append(net_desc) +// spec['nic_count'] = len(net_info) +// spec['manufacture'] = self.sys_info.get('manufacture', 'Unknown') +// spec['model'] = self.sys_info.get('model', 'Unknown') +// return spec +// def get_hardware_specification(self): +// spec = {} +// spec['cpu'] = self.cpu_count +// spec['mem'] = self.mem_size +// if self.storage_info: +// from clouds.baremetal import diskconfig +// spec['disk'] = diskconfig.get_disk_spec_v2(self.storage_info) +// spec['driver'] = self.storage_driver +// spec['storage_info'] = self.storage_info +// return spec + +func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { + if statusCheck { + if utils.IsInStringArray(self.Status, []string{BAREMETAL_INIT, BAREMETAL_PREPARE_FAIL, BAREMETAL_PREPARE}) || + self.getBaremetalServer() != nil { + return nil + } + if self.MemSize == 0 || self.CpuCount == 0 { + return nil + } + } + spec := self.GetHardwareSpecification() + spec.Remove("storage_info") + netInfo := jsonutils.NewArray() + nifs := self.GetNetInterfaces() + for _, nif := range nifs { + netDesc := nif.getBaremetalJsonDesc() + nicType, err := netDesc.GetString("nic_type") + if err != nil && nicType != NIC_TYPE_IPMI { + netInfo.Add(netDesc) + } + } + spec.Set("nic_count", jsonutils.NewInt(int64(netInfo.Length()))) + manufacture, err := self.SysInfo.Get("manufacture") + if err != nil { + manufacture = jsonutils.NewString("Unknown") + } + spec.Set("manufacture", manufacture) + model, err := self.SysInfo.Get("model") + if err != nil { + model = jsonutils.NewString("Unknown") + } + spec.Set("model", model) + return nil +} + +func (self *SHost) GetSpecIdent(spec *jsonutils.JSONDict) []string { + // Todo + return []string{} +} + +func (self *SHost) GetHardwareSpecification() *jsonutils.JSONDict { + spec := jsonutils.NewDict() + spec.Set("cpu", jsonutils.NewInt(int64(self.CpuCount))) + spec.Set("mem", jsonutils.NewInt(int64(self.MemSize))) + if self.StorageInfo != nil { + spec.Set("disk", GetDiskSpecV2(self.StorageInfo)) + spec.Set("driver", jsonutils.NewString(self.StorageDriver)) + spec.Set("storage_info", self.StorageInfo) + } + return spec +} + type SStorageCapacity struct { Capacity int Used int diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index f3182d3f67..d1c7c26c21 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -380,6 +380,24 @@ func (self *SIsolatedDevice) getDesc() *jsonutils.JSONDict { return desc } +func (self *SIsolatedDevice) GetSpec(checkStatus bool) *jsonutils.JSONDict { + if checkStatus { + if len(self.GuestId) > 0 { + return nil + } + host := self.getHost() + if host.Status != BAREMETAL_RUNNING || !host.Enabled { + return nil + } + } + spec := jsonutils.NewDict() + spec.Set("dev_type", jsonutils.NewString(self.DevType)) + spec.Set("model", jsonutils.NewString(self.Model)) + spec.Set("pci_id", jsonutils.NewString(self.VendorDeviceId)) + spec.Set("vendor", jsonutils.NewString(self.getVendor())) + return spec +} + func (self *SIsolatedDevice) GetShortDesc() *jsonutils.JSONDict { desc := self.getDesc() desc.Add(jsonutils.NewString(self.Keyword()), "res_name") From d113017c338c3a0b7bf68960e624aa4d45b5f606 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Fri, 24 Aug 2018 22:41:24 +0800 Subject: [PATCH 20/45] remove temp code --- pkg/compute/models/guests.go | 25 ------------------------- pkg/compute/models/hosts.go | 31 ------------------------------- 2 files changed, 56 deletions(-) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index aef8301dc4..713f5e88c9 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3095,31 +3095,6 @@ func (self *SGuest) GetTemplateId() string { return "" } -// def get_short_desc(self, user_cred=None): -// desc = super(Guests, self).get_short_desc() -// desc['mem'] = self.vmem_size -// desc['cpu'] = self.vcpu_count -// template_id = self.get_template_id() -// if template_id is not None: -// desc['template_id'] = template_id -// ext_bw, int_bw = self.get_bandwidth() -// if ext_bw > 0: -// desc['ext_bandwidth'] = ext_bw -// if int_bw > 0: -// desc['int_bandwidth'] = int_bw -// hypervisor = self.get_hypervisor() -// desc['hypervisor'] = hypervisor -// spec = self.get_spec(user_cred, status_check=False) -// if hypervisor == self.HYPERVISOR_BAREMETAL: -// host = self.get_host() -// if host is not None: -// host_spec = '/'.join(host.get_spec_ident( -// host.get_spec(user_cred, status_check=False))) -// spec['host_spec'] = host_spec -// if spec is not None: -// desc.update(spec) -// return desc - func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { desc := self.SStandaloneResourceBase.GetShortDesc() desc.Set("mem", jsonutils.NewInt(int64(self.VmemSize))) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 97ccecb6ef..0874133ba1 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -506,37 +506,6 @@ func (self *SHost) ClearSchedDescCache() error { return HostManager.ClearSchedDescCache(self.Id) } -// def get_spec(self, user_cred, status_check=True, **kwargs): -// if status_check: -// if self.status in [baremetalstatus.INIT, baremetalstatus.PREPARE_FAIL, -// baremetalstatus.PREPARE] or \ -// self.get_baremetal_server() is not None: -// return None -// if self.mem_size is None or self.cpu_count is None: -// return None -// from clouds.baremetal import nictypes -// spec = self.get_hardware_specification() -// spec.pop('storage_info', None) -// net_info = [] -// for netif in self.get_netifs(): -// net_desc = netif.get_baremetal_json_desc() -// if net_desc.get('nic_type', None) != nictypes.NIC_TYPE_IPMI: -// net_info.append(net_desc) -// spec['nic_count'] = len(net_info) -// spec['manufacture'] = self.sys_info.get('manufacture', 'Unknown') -// spec['model'] = self.sys_info.get('model', 'Unknown') -// return spec -// def get_hardware_specification(self): -// spec = {} -// spec['cpu'] = self.cpu_count -// spec['mem'] = self.mem_size -// if self.storage_info: -// from clouds.baremetal import diskconfig -// spec['disk'] = diskconfig.get_disk_spec_v2(self.storage_info) -// spec['driver'] = self.storage_driver -// spec['storage_info'] = self.storage_info -// return spec - func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { if statusCheck { if utils.IsInStringArray(self.Status, []string{BAREMETAL_INIT, BAREMETAL_PREPARE_FAIL, BAREMETAL_PREPARE}) || From b16613a0e6de1fcacf9c42e438133ca9d3b2f05f Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 25 Aug 2018 09:24:05 +0800 Subject: [PATCH 21/45] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E9=98=BF?= =?UTF-8?q?=E9=87=8C=E4=BA=91=E4=B8=BB=E6=9C=BA=E5=88=9B=E5=BB=BA=E5=90=8E?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=90=AF=E5=8A=A8=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/aliyun.go | 9 +++++---- pkg/compute/tasks/guest_create_task.go | 4 ++-- pkg/compute/tasks/guest_start_task.go | 1 + 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 50ebab0f8d..673c4f1b1b 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -125,10 +125,11 @@ type SDiskInfo struct { func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) - onfinish, err := config.GetString("on_finish") + /* onfinish, err := config.GetString("on_finish") if err != nil { return err - } + } */ + action, err := config.GetString("action") if err != nil { return err @@ -170,12 +171,12 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return nil, err } - if onfinish == "none" { + /* if onfinish == "none" { err = iVM.StartVM() if err != nil { return nil, err } - } + } */ encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) if err != nil { diff --git a/pkg/compute/tasks/guest_create_task.go b/pkg/compute/tasks/guest_create_task.go index a57f49afca..1248fe287e 100644 --- a/pkg/compute/tasks/guest_create_task.go +++ b/pkg/compute/tasks/guest_create_task.go @@ -105,7 +105,7 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DEPLOY_FAILED, data.String()) } -func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { +/*func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) } @@ -113,4 +113,4 @@ func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStand func (self *GuestCreateTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) -} +}*/ diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go index 3d910543a6..8be672f918 100644 --- a/pkg/compute/tasks/guest_start_task.go +++ b/pkg/compute/tasks/guest_start_task.go @@ -72,6 +72,7 @@ func (self *GuestStartTask) OnStartCompleteFailed(ctx context.Context, obj db.IS } func (self *GuestStartTask) onStartGuestFailed(ctx context.Context, guest *models.SGuest, err error) { + guest.SetStatus(self.UserCred, models.VM_START_FAILED, err.Error()) self.SetStageFailed(ctx, err.Error()) self.OnStartCompleteFailed(ctx, guest, jsonutils.NewString(err.Error())) } From ed2bf2eba5817476dbccbf17d68cb18a454a2804 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 09:11:45 +0000 Subject: [PATCH 22/45] dns: policy, log, db query optimization Fixes CLOUD-451 TODO - timeout bound for k8s query - k8s client pools --- pkg/dns/dns.go | 161 +++++++++++++++++++++------------------------ pkg/dns/parse.go | 49 +++++++------- pkg/dns/reverse.go | 12 +--- pkg/dns/setup.go | 9 +-- 4 files changed, 107 insertions(+), 124 deletions(-) diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index b60ab73c51..9485a2557f 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -18,25 +18,27 @@ import ( "github.com/mholt/caddy" "github.com/miekg/dns" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" ylog "yunion.io/x/log" - "yunion.io/x/pkg/utils" - "yunion.io/x/sqlchemy" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/k8s" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" ) const ( PluginName string = "yunion" // defaultTTL to apply to all answers - defaultTTL = 10 + defaultTTL = 10 + defaultDbMaxOpenConn = 32 + defaultDbMaxIdleConn = 32 ) var ( @@ -74,15 +76,17 @@ func (r *SRegionDNS) initDB(c *caddy.Controller) error { if err != nil { return err } - dbConn, err := sql.Open(dialect, sqlStr) + sqlDb, err := sql.Open(dialect, sqlStr) if err != nil { return err } - sqlchemy.SetDB(dbConn) + sqlDb.SetMaxOpenConns(defaultDbMaxOpenConn) + sqlDb.SetMaxIdleConns(defaultDbMaxIdleConn) + sqlchemy.SetDB(sqlDb) db.InitAllManagers() c.OnShutdown(func() error { - r.CloseDB() + sqlchemy.CloseDB() return nil }) return nil @@ -103,10 +107,6 @@ func (r *SRegionDNS) initK8s(c *caddy.Controller) { ylog.Infof("Init k8s client success, %d pods in the cluster", len(pods.Items)) } -func (r *SRegionDNS) CloseDB() { - sqlchemy.CloseDB() -} - func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *dns.Msg) (int, error) { var ( records []dns.RR @@ -116,36 +116,26 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d opt := plugin.Options{} state := request.Request{W: w, Req: rmsg, Context: ctx} - zone := plugin.Zones(r.Zones).Matches(state.Name()) - switch state.QType() { case dns.TypeA: - ylog.Debugf("A question: %#v", state) records, err = plugin.A(r, zone, state, nil, opt) case dns.TypeAAAA: - ylog.Debugf("AAAA question: %#v", state) + // TODO fallthrough to next records, err = plugin.AAAA(r, zone, state, nil, opt) case dns.TypeTXT: - ylog.Debugf("TXT question: %#v", state) records, err = plugin.TXT(r, zone, state, opt) case dns.TypeCNAME: - ylog.Debugf("CNAME question: %#v", state) records, err = plugin.CNAME(r, zone, state, opt) case dns.TypePTR: - ylog.Debugf("PTR question: %#v", state) records, err = plugin.PTR(r, zone, state, opt) case dns.TypeMX: - ylog.Debugf("MX question: %#v", state) records, extra, err = plugin.MX(r, zone, state, opt) case dns.TypeSRV: - ylog.Debugf("SRV question: %#v", state) records, extra, err = plugin.SRV(r, zone, state, opt) case dns.TypeSOA: - ylog.Debugf("SOA question: %#v", state) records, err = plugin.SOA(r, zone, state, opt) case dns.TypeNS: - ylog.Debugf("NS question: %#v", state) if state.Name() == zone { records, extra, err = plugin.NS(r, zone, state, opt) break @@ -157,18 +147,19 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d _, err = plugin.A(r, zone, state, nil, opt) } - if r.IsNameError(err) { + if err == errCallNext { if r.Fall.Through(state.Name()) { return plugin.NextOrFailure(r.Name(), r.Next, ctx, w, rmsg) } return plugin.BackendError(r, zone, dns.RcodeNameError, state, nil /* err */, opt) - } - if err != nil { - return plugin.BackendError(r, zone, dns.RcodeServerFailure, state, err, opt) + } else if err == errRefused { + return plugin.BackendError(r, zone, dns.RcodeRefused, state, err, opt) + } else if err == errNotFound { + return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt) } if len(records) == 0 { - return plugin.BackendError(r, zone, dns.RcodeSuccess, state, err, opt) + return plugin.BackendError(r, zone, dns.RcodeNameError, state, err, opt) } m := new(dns.Msg) @@ -184,13 +175,14 @@ func (r *SRegionDNS) ServeDNS(ctx context.Context, w dns.ResponseWriter, rmsg *d } var ( - errNoItems = errors.New("no items found") + errRefused = errors.New("refused the query") + errNotFound = errors.New("not found") + errCallNext = errors.New("continue to next") ) // Services implements the ServiceBackend interface func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Options) (services []msg.Service, err error) { switch state.QType() { - case dns.TypeTXT: t, _ := dnsutil.TrimZone(state.Name(), state.Zone) @@ -203,7 +195,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti } svc := msg.Service{Text: "0.0.1", TTL: 28800, Key: msg.Path(state.QName(), "coredns")} return []msg.Service{svc}, nil - case dns.TypeNS: ns := r.nsAddr() svc := msg.Service{Host: ns.A.String(), Key: msg.Path(state.QName(), "coredns")} @@ -219,7 +210,6 @@ func (r *SRegionDNS) Services(state request.Request, exact bool, opt plugin.Opti } services, err = r.Records(state, false) - ylog.Debugf("Get records: %#v, error: %v", services, err) return } @@ -230,7 +220,7 @@ func (r *SRegionDNS) Lookup(state request.Request, name string, typ uint16) (*dn // IsNameError implements the ServiceBackend interface func (r *SRegionDNS) IsNameError(err error) bool { - return err == errNoItems + return err == errCallNext } // Records looks up records in region mysql @@ -242,25 +232,22 @@ func (r *SRegionDNS) Records(state request.Request, exact bool) ([]msg.Service, return r.findRecords(req) } -func (r *SRegionDNS) getHostIpWithName(req *recordRequest) []string { +func (r *SRegionDNS) getHostIpWithName(req *recordRequest) string { name := req.QueryName() host, _ := models.HostManager.FetchByName("", name) if host == nil { - return nil + return "" } ip := host.(*models.SHost).AccessIp - if len(ip) == 0 { - return nil - } - return []string{ip} + return ip } func (r *SRegionDNS) getGuestIpWithName(req *recordRequest) []string { ips := []string{} name := req.QueryName() projectId := req.ProjectId() - isExitOnly := req.IsExitOnly() - ips = models.GuestManager.GetIpInProjectWithName(projectId, name, isExitOnly) + wantOnlyExit := false + ips = models.GuestManager.GetIpInProjectWithName(projectId, name, wantOnlyExit) return ips } @@ -268,6 +255,9 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error) queryInfo := req.GetK8sQueryInfo() pods, err := r.getK8sServicePods(queryInfo.Namespace, queryInfo.ServiceName) if err != nil { + if k8serrors.IsNotFound(err) { + err = nil + } return nil, err } ips := make([]string, 0) @@ -301,10 +291,9 @@ func (r *SRegionDNS) Name() string { return PluginName } -func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service, err error) { +func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Service) { ips := models.DnsRecordManager.QueryDnsIps(req.ProjectId(), req.Name(), req.Type()) if len(ips) == 0 { - err = errNoItems return } @@ -317,12 +306,12 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic if req.IsSRV() { parts := strings.SplitN(ip.Addr, ":", 2) if len(parts) != 2 { - err = fmt.Errorf("Invalid SRV records: %q", ip.Addr) + ylog.Errorf("Invalid SRV records: %q", ip.Addr) return } port, e := strconv.Atoi(parts[1]) if e != nil { - err = e + ylog.Errorf("Invalid SRV records: %q", ip.Addr) return } s = msg.Service{Host: parts[0], Port: port, TTL: ttl} @@ -334,17 +323,6 @@ func (r *SRegionDNS) queryLocalDnsRecords(req *recordRequest) (recs []msg.Servic return } -func (r *SRegionDNS) IsCloudNetworkIp(req *recordRequest) bool { - if req.network != nil { - return true - } - return false -} - -func (r *SRegionDNS) IsK8sClientReady() bool { - return r.K8sClient != nil -} - func (r *SRegionDNS) isMyDomain(req *recordRequest) bool { zones := []string{fmt.Sprintf("%s.", r.PrimaryZone)} zone := plugin.Zones(zones).Matches(req.state.Name()) @@ -354,54 +332,65 @@ func (r *SRegionDNS) isMyDomain(req *recordRequest) bool { return false } -func (r *SRegionDNS) findRecords(req *recordRequest) (recs []msg.Service, err error) { +func (r *SRegionDNS) findRecords(req *recordRequest) ([]msg.Service, error) { // 1. try local dns records table - recs, err = r.queryLocalDnsRecords(req) - if len(recs) != 0 { - return + rrs := r.queryLocalDnsRecords(req) + if len(rrs) > 0 { + return rrs, nil } + isPlainName := req.IsPlainName() isMyDomain := r.isMyDomain(req) - - isCloudIp := r.IsCloudNetworkIp(req) - - // 2. not my domain and src ip not in cloud network table - // query from upstream - if !isMyDomain && !isCloudIp { - err = errNoItems - return + if isPlainName { + isCloudIp := req.SrcInCloud() + if isCloudIp { + ips := r.findInternalRecordIps(req) + if len(ips) > 0 { + return ips2DnsRecords(ips), nil + } else { + return nil, errNotFound + } + } else { + return nil, errRefused + } + } else if isMyDomain { + ips := r.findInternalRecordIps(req) + if len(ips) > 0 { + return ips2DnsRecords(ips), nil + } else { + return nil, errNotFound + } + } else { + return nil, errCallNext } - - // 3. internal query - ips, err := r.findInternalRecordIps(req) - return ips2DnsRecords(ips), err } -func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) ([]string, error) { - // 1. try host table - ip := r.getHostIpWithName(req) - if len(ip) != 0 { - return ip, nil +func (r *SRegionDNS) findInternalRecordIps(req *recordRequest) []string { + { + // 1. try host table + ip := r.getHostIpWithName(req) + if len(ip) > 0 { + return []string{ip} + } } - // 2. try guest table - ip = r.getGuestIpWithName(req) - if len(ip) != 0 { - return ip, nil + { + // 2. try guest table + ips := r.getGuestIpWithName(req) + if len(ips) > 0 { + return ips + } } - if !r.IsK8sClientReady() { + if r.K8sClient == nil { ylog.Warningf("K8s client not ready, skip it.") - return nil, errNoItems + return nil } // 3. try k8s service backends ips, err := r.getK8sServiceBackends(req) - if len(ips) != 0 { - return ips, nil - } if err != nil { ylog.Errorf("Get k8s service backends error: %v", err) } - return nil, errNoItems + return ips } func ips2DnsRecords(ips []string) []msg.Service { diff --git a/pkg/dns/parse.go b/pkg/dns/parse.go index 5cf836b646..568a773567 100644 --- a/pkg/dns/parse.go +++ b/pkg/dns/parse.go @@ -7,17 +7,16 @@ import ( "github.com/coredns/coredns/request" "github.com/miekg/dns" - "yunion.io/x/pkg/tristate" - "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/pkg/tristate" ) type recordRequest struct { - state request.Request - domainSegs []string - guest *models.SGuest - host *models.SHost - network *models.SNetwork + state request.Request + domainSegs []string + srcProjectId string + srcInCloud bool + network *models.SNetwork } func parseRequest(state request.Request) (r *recordRequest, err error) { @@ -28,9 +27,19 @@ func parseRequest(state request.Request) (r *recordRequest, err error) { domainSegs: segs, } srcIP := r.SrcIP4() - r.guest = models.GuestnetworkManager.GetGuestByAddress(srcIP) - r.host = models.HostnetworkManager.GetHostByAddress(srcIP) - r.network, _ = models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None) + // NOTE the check on networks_tbl is a hack, we should be more specific + // by querying only guest networks, host networks, and others like + // loadbalancer network the to come. + // + // Order matters here, we want to find the srcIP project as accurately + // as possible + if guest := models.GuestnetworkManager.GetGuestByAddress(srcIP); guest != nil { + r.srcProjectId = guest.ProjectId + r.srcInCloud = true + } else if network, _ := models.NetworkManager.GetNetworkOfIP(srcIP, "", tristate.None); network != nil { + r.srcProjectId = network.ProjectId + r.srcInCloud = true + } return } @@ -41,6 +50,11 @@ func (r recordRequest) Name() string { return name } +func (r recordRequest) IsPlainName() bool { + nl := dns.CountLabel(r.Name()) + return nl == 1 +} + func (r recordRequest) QueryName() string { seps := strings.Split(r.Name(), ".") if len(seps) == 0 { @@ -63,20 +77,11 @@ func (r recordRequest) SrcIP4() string { } func (r recordRequest) ProjectId() string { - if r.guest != nil { - return r.guest.ProjectId - } - if r.network != nil { - return r.network.ProjectId - } - return "" + return r.srcProjectId } -func (r recordRequest) IsExitOnly() bool { - if r.guest == nil { - return false - } - return r.guest.IsExitOnly() +func (r recordRequest) SrcInCloud() bool { + return r.srcInCloud } type K8sQueryInfo struct { diff --git a/pkg/dns/reverse.go b/pkg/dns/reverse.go index e6cf7e2276..f6df515fdc 100644 --- a/pkg/dns/reverse.go +++ b/pkg/dns/reverse.go @@ -8,8 +8,6 @@ import ( "github.com/coredns/coredns/plugin/pkg/dnsutil" "github.com/coredns/coredns/request" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/compute/models" ) @@ -21,13 +19,7 @@ func (r *SRegionDNS) Reverse(state request.Request, exact bool, opt plugin.Optio return nil, e } records, err := r.getNameForIp(ip, state) - if err != nil { - log.Errorf("Reverse get name for ip: %v", err) - } - if len(records) == 0 { - return records, errNoItems - } - return records, nil + return records, err } func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Service, error) { @@ -53,7 +45,7 @@ func (r *SRegionDNS) getNameForIp(ip string, state request.Request) ([]msg.Servi if guest != nil { return []msg.Service{{Host: r.joinDomain(guest.Name), TTL: defaultTTL}}, nil } - return nil, errNoItems + return nil, errNotFound } func (r *SRegionDNS) joinDomain(name string) string { diff --git a/pkg/dns/setup.go b/pkg/dns/setup.go index 1a6a1ff673..77a9cb1c4b 100644 --- a/pkg/dns/setup.go +++ b/pkg/dns/setup.go @@ -2,7 +2,6 @@ package dns import ( "fmt" - "os" "github.com/coredns/coredns/core/dnsserver" "github.com/coredns/coredns/plugin" @@ -20,18 +19,16 @@ func init() { } func setup(c *caddy.Controller) error { - os.Stderr = os.Stdout - rDNS, err := regionDNSParse(c) if err != nil { return plugin.Error(PluginName, err) } - if rDNS.PrimaryZone == "" { - return fmt.Errorf("dns_domain must provided") + if len(rDNS.PrimaryZone) == 0 { + return fmt.Errorf("dns_domain missing") } if !regutils.MatchDomainName(rDNS.PrimaryZone) { - return fmt.Errorf("dns_domain %q not match domain format", rDNS.PrimaryZone) + return fmt.Errorf("dns_domain %q invalid", rDNS.PrimaryZone) } err = rDNS.initDB(c) From 5b12bdc7e75f769f078eaf2942132fd2e14bf1a0 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 10:12:43 +0000 Subject: [PATCH 23/45] dns: hack: new k8s client for each query - rest.Config Timeout setting does not work right now [1] - number of connections will not increase along with newed client: they use a single connection always... [1] https://github.com/kubernetes/client-go/issues/374#issuecomment-413842964 --- pkg/dns/dns.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index 9485a2557f..248504b9f6 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -271,7 +271,11 @@ func (r *SRegionDNS) getK8sServiceBackends(req *recordRequest) ([]string, error) } func (r *SRegionDNS) getK8sServicePods(namespace, name string) ([]v1.Pod, error) { - cli := r.K8sClient + cli, err := k8s.NewClientByFile(r.K8sConfigFile, nil) + if err != nil { + ylog.Errorf("Init kubernetes client error: %v", err) + return nil, err + } svc, err := cli.CoreV1().Services(namespace).Get(name, metav1.GetOptions{}) if err != nil { return nil, err From 96d52d1061bc8d573b154a97e217acc641429885 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 25 Aug 2018 11:02:03 +0800 Subject: [PATCH 24/45] =?UTF-8?q?=E4=BF=AE=E6=AD=A3:=20task=20=E5=9B=9E?= =?UTF-8?q?=E8=B0=83=E4=B8=8D=E5=88=B0=20stage=20tailed=20=E7=9A=84?= =?UTF-8?q?=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/appsrv/dispatcher/dispatcher.go | 2 +- pkg/cloudcommon/db/taskman/tasks.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/appsrv/dispatcher/dispatcher.go b/pkg/appsrv/dispatcher/dispatcher.go index 56cb359b66..89048c175f 100644 --- a/pkg/appsrv/dispatcher/dispatcher.go +++ b/pkg/appsrv/dispatcher/dispatcher.go @@ -203,7 +203,7 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht manager, params, query, body := fetchEnv(ctx, w, r) data, _ := body.Get(manager.KeywordPlural()) if data == nil { - data = jsonutils.NewDict() + data = body.(*jsonutils.JSONDict) } results, err := manager.PerformClassAction(ctx, params[""], query, data) if err != nil { diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index aa08e2a1e8..277032c453 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -296,7 +296,7 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, taskStatus, _ := data.GetString("__status__") if len(taskStatus) > 0 && taskStatus != "OK" { taskFailed = true - data, err = data.Get("reason") + data, err = data.Get("__reason__") if err != nil { data = jsonutils.NewString("Task failed due to unknown remote errors!") } From 43d8b74d175144ac086f6bf3849727b292a77726 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 24 Aug 2018 12:00:18 +0000 Subject: [PATCH 25/45] dns: add a README.md for memo purposes --- pkg/dns/README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 pkg/dns/README.md diff --git a/pkg/dns/README.md b/pkg/dns/README.md new file mode 100644 index 0000000000..79bfeb66f3 --- /dev/null +++ b/pkg/dns/README.md @@ -0,0 +1,56 @@ +# 行为约定 + +A + +```sh +names='' +names="$names titan" #ok, guest PlainName +names="$names titan.hq.cloud.yunionyun.com" #ok, guest CloudZoneFQDN +names="$names kubenode" #ok, host PlainName +names="$names kubenode.hq.cloud.yunionyun.com" #ok, host CloudZoneFQDN +names="$names whoever-the-ether" #NXDOMAIN, nonexistent PlainName +names="$names whoever-the-ether.titan.hq.cloud.yunionyun.com" #NXDOMAIN, nonexistent CloudZoneFQDN +names="$names mail.google.com" #ok, dnsrecords +names="$names www.douban.com" #ok, pub +names="$names app" #ok, k8s svc PlainName in "default" ns +names="$names app.default" #NXDOMAIN, k8s svc name.namespace +names="$names app.default.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN +names="$names mon-kafka.system" #NXDOMAIN, k8s svc name.namespace +names="$names mon-kafka.system.hq.cloud.yunionyun.com" #ok, k8s name.ns CloudZoneFQDN + +# TODO source IP +# TODO other TYPEs +for name in $names; do + echo "############### $name" + #dig @192.168.222.171 $name + #dig -p 54 @10.168.222.136 $name + dig -p 54 @192.168.222.171 $name +done +``` + +PTR + +```sh +names='' +names="$names " #ok, guest ip +names="$names " #ok, host ip +names="$names " #ok, ptr records in db +names="$names " #NXDOMAIN, others + +for name in $names; do + echo "############### $name" + dig -p 54 @192.168.222.171 -x $name +done +``` + +# 配置 + + log { + # note that apart from rcode like NXDOMAIN, SERVFAIL, coredns will also + # log NOERROR response when it's NoData as defined by coredns itself + # + # > NoData indicates name found, but not the type: NOERROR in header, SOA in auth. + # + class denial + class error + } From 3be82ebd540e75a4c690dae9e686c6777f527851 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Sat, 25 Aug 2018 04:37:48 +0000 Subject: [PATCH 26/45] guestnetworks, hostnetworks: avoid subquery in get by address call --- pkg/compute/models/guestnetworks.go | 3 ++- pkg/compute/models/hostnetworks.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 2f3defdf37..223d6f4d4d 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -263,9 +263,10 @@ func (self *SGuestnetwork) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { } func (manager *SGuestnetworkManager) GetGuestByAddress(address string) *SGuest { - networks := manager.Query().SubQuery() + networks := manager.TableSpec().Instance() guests := GuestManager.Query() q := guests.Join(networks, sqlchemy.AND( + sqlchemy.IsFalse(networks.Field("deleted")), sqlchemy.Equals(networks.Field("ip_addr"), address), sqlchemy.Equals(networks.Field("guest_id"), guests.Field("id")), )) diff --git a/pkg/compute/models/hostnetworks.go b/pkg/compute/models/hostnetworks.go index 88433fb90a..5a94918e32 100644 --- a/pkg/compute/models/hostnetworks.go +++ b/pkg/compute/models/hostnetworks.go @@ -107,9 +107,10 @@ func (man *SHostnetworkManager) GetNetworkByAddress(addr string) *SNetwork { } func (man *SHostnetworkManager) GetHostByAddress(addr string) *SHost { - networks := man.Query().SubQuery() + networks := man.TableSpec().Instance() hosts := HostManager.Query() q := hosts.Join(networks, sqlchemy.AND( + sqlchemy.IsFalse(networks.Field("deleted")), sqlchemy.Equals(networks.Field("ip_addr"), addr), sqlchemy.Equals(networks.Field("baremetal_id"), hosts.Field("id")), )) From 40b99ef250b41dfc0f8b0853f18aa76210bac93b Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 25 Aug 2018 12:50:41 +0800 Subject: [PATCH 27/45] performClassActionHandler: handle body is nil --- pkg/appsrv/dispatcher/dispatcher.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/appsrv/dispatcher/dispatcher.go b/pkg/appsrv/dispatcher/dispatcher.go index de313f451c..ea95a458f9 100644 --- a/pkg/appsrv/dispatcher/dispatcher.go +++ b/pkg/appsrv/dispatcher/dispatcher.go @@ -209,9 +209,8 @@ func performClassActionHandler(ctx context.Context, w http.ResponseWriter, r *ht if data == nil { data = body.(*jsonutils.JSONDict) } - } - if data == nil { - data = body.(*jsonutils.JSONDict) + } else { + data = jsonutils.NewDict() } results, err := manager.PerformClassAction(ctx, params[""], query, data) if err != nil { From a39eea4b646da92b2e4df6ceb05980c4849164e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 12:57:42 +0800 Subject: [PATCH 28/45] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E5=8F=AF=E7=94=A8storage=E6=97=B6=EF=BC=8C=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E6=95=B0=E6=8D=AEbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/storages.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 7990188dd8..769a76612d 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -664,12 +664,15 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. } if jsonutils.QueryBoolean(query, "usable", false) { - hostStorage := HoststorageManager.Query().SubQuery() - q = q.Join(hostStorage, sqlchemy.AND( - sqlchemy.Equals(hostStorage.Field("storage_id"), q.Field("id")), - sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE}), - sqlchemy.IsTrue(q.Field("enabled")), - )) + hostStorageTable := HoststorageManager.Query().SubQuery() + hostTable := HostManager.Query().SubQuery() + sq := hostStorageTable.Query(hostStorageTable.Field("storage_id")).Join(hostTable, + sqlchemy.Equals(hostTable.Field("id"), hostStorageTable.Field("host_id"))). + Filter(sqlchemy.Equals(hostTable.Field("host_status"), HOST_ONLINE)) + + q = q.Filter(sqlchemy.In(q.Field("id"), sq)). + Filter(sqlchemy.In(q.Field("status"), []string{STORAGE_ENABLED, STORAGE_ONLINE})). + Filter(sqlchemy.IsTrue(q.Field("enabled"))) } managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) From 4f24cbe615573ecb0b61ad59726c03349ec020e7 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 25 Aug 2018 13:07:25 +0800 Subject: [PATCH 29/45] remove some debug log --- pkg/cloudcommon/db/db_dispatcher.go | 4 ++-- pkg/cloudcommon/db/virtualresource.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index fe06fc2f9d..4541621ca1 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -524,7 +524,7 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr } func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - log.Debugf("Get %s", idStr) + // log.Debugf("Get %s", idStr) userCred := fetchUserCredential(ctx) data, err := dispatcher.tryGetModelProperty(ctx, idStr, query) @@ -541,7 +541,7 @@ func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, quer } else if err != nil { return nil, err } - log.Debugf("Get found %s", model) + // log.Debugf("Get found %s", model) if !model.AllowGetDetails(ctx, userCred, query) { return nil, httperrors.NewForbiddenError("Not allow to get details") } diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index 28360ccb55..f811e434a0 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -161,16 +161,16 @@ func (model *SVirtualResourceBase) AllowPerformMetadata(ctx context.Context, use } func (model *SVirtualResourceBase) GetTenantCache(ctx context.Context) (*STenant, error) { - log.Debugf("Get tenant by Id %s", model.ProjectId) + // log.Debugf("Get tenant by Id %s", model.ProjectId) return TenantCacheManager.FetchTenantById(ctx, model.ProjectId) } func (model *SVirtualResourceBase) getMoreDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, extra *jsonutils.JSONDict) *jsonutils.JSONDict { if userCred.IsSystemAdmin() { - log.Debugf("GetCustomizeColumns") + // log.Debugf("GetCustomizeColumns") tobj, err := model.GetTenantCache(ctx) if err == nil { - log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj)) + // log.Debugf("GetTenantFromCache %s", jsonutils.Marshal(tobj)) extra.Add(jsonutils.NewString(tobj.GetName()), "tenant") } else { log.Errorf("GetTenantCache fail %s", err) From 291cf2547e87440a14bc67c04af76e67b01ba2e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 13:36:27 +0800 Subject: [PATCH 30/45] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddisk-resize=E5=90=8E?= =?UTF-8?q?=E6=9C=AA=E6=AD=A3=E7=A1=AE=E8=AE=BE=E7=BD=AEguest=E7=8A=B6?= =?UTF-8?q?=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/tasks/disk_resize_task.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/compute/tasks/disk_resize_task.go b/pkg/compute/tasks/disk_resize_task.go index 584c43ab94..dd25da5879 100644 --- a/pkg/compute/tasks/disk_resize_task.go +++ b/pkg/compute/tasks/disk_resize_task.go @@ -32,7 +32,7 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, } resion := "Cannot find host for disk" if host == nil || host.HostStatus != models.HOST_ONLINE { - disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion) + disk.SetDiskReady(ctx, self.GetUserCred(), resion) self.SetStageFailed(ctx, resion) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion, self.GetUserCred()) } else { @@ -65,7 +65,7 @@ func (self *DiskResizeTask) OnStartResizeDiskSucc(ctx context.Context, disk *mod } func (self *DiskResizeTask) OnStartResizeDiskFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.GetUserCred(), models.DISK_READY, resion.Error()) + disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error()) self.SetStageFailed(ctx, resion.Error()) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, resion.Error(), self.GetUserCred()) } @@ -94,6 +94,7 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode self.OnStartResizeDiskFailed(ctx, disk, err) return } + disk.SetDiskReady(ctx, self.GetUserCred(), "") notes := fmt.Sprintf("%s=>%s", oldStatus, disk.Status) db.OpsLog.LogEvent(disk, db.ACT_UPDATE_STATUS, notes, self.UserCred) self.CleanHostSchedCache(disk) @@ -103,6 +104,6 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode } func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.UserCred, models.DISK_READY, resion.Error()) + disk.SetDiskReady(ctx, self.GetUserCred(), resion.Error()) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred) } From 4014c55b69a817cee771ec3415d96330c2263a4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 13:39:54 +0800 Subject: [PATCH 31/45] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9C=AA=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96guestmanager=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/disks.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 047863499a..198586a6ee 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -560,9 +560,8 @@ func (self *SDisk) GetAttachedGuests() []SGuest { q = q.Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)) ret := make([]SGuest, 0) - err := q.All(&ret) - if err != nil { - log.Errorf("%s", err) + if err := db.FetchModelObjects(GuestManager, q, &ret); err != nil { + log.Errorf("Fetch Geusts Objects %v", err) return nil } return ret From 2914e880af453cf59f793216268dbc5f0ddfe562 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 25 Aug 2018 14:31:05 +0800 Subject: [PATCH 32/45] fix: NotifyParentTaskFailure reason --- pkg/cloudcommon/db/taskman/tasks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index 277032c453..de893d3476 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -538,7 +538,7 @@ func (self *STask) NotifyParentTaskFailure(ctx context.Context, reason string) { if len(reason) > 100 { reason = reason[:100] + "..." } - body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason))) + body.Add(jsonutils.NewString(fmt.Sprintf("Subtask %s failed: %s", self.TaskName, reason)), "__reason__") self.NotifyParentTaskComplete(ctx, body, true) } From 618f183428212af860a4e6b62446f82214af5254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 14:45:45 +0800 Subject: [PATCH 33/45] =?UTF-8?q?release/2.0.0=E7=89=88=E6=9C=AC=E9=9C=80?= =?UTF-8?q?=E8=A6=81disk=E8=AF=A6=E6=83=85=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/disks.go | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 198586a6ee..699556c1bd 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -16,6 +16,7 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/sysutils" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -151,6 +152,21 @@ func (self *SDisk) GetGuestdisks() []SGuestdisk { } return guestdisks } +func (self *SDisk) GetGuests() []SGuest { + result := make([]SGuest, 0) + query := GuestManager.Query() + guestdisks := GuestdiskManager.Query().SubQuery() + q := query.Join(guestdisks, sqlchemy.AND( + sqlchemy.Equals(guestdisks.Field("guest_id"), query.Field("id")))). + Filter(sqlchemy.Equals(guestdisks.Field("disk_id"), self.Id)) + // q.DebugQuery() + err := db.FetchModelObjects(GuestManager, q, &result) + if err != nil { + log.Errorf(err.Error()) + return nil + } + return result +} func (self *SDisk) ValidateDeleteCondition(ctx context.Context) error { if self.GetGuestDiskCount() > 0 { @@ -183,6 +199,13 @@ func (self *SDisk) GetStorage() *SStorage { return nil } +func (self *SDisk) GetCloudprovider() *SCloudprovider { + if storage := self.GetStorage(); storage != nil { + return storage.GetCloudprovider() + } + return nil +} + func (self *SDisk) GetPathAtHost(host *SHost) string { storage := self.GetStorage() if storage.StorageType == STORAGE_RBD { @@ -536,6 +559,49 @@ func (self *SDisk) CustomizeDelete(ctx context.Context, userCred mcclient.TokenC return self.StartDiskDeleteTask(ctx, userCred, "", false) } +func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + } + if storage := self.GetStorage(); storage != nil { + extra.Add(jsonutils.NewString(storage.GetName()), "storage") + extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") + extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") + extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") + if zone := storage.getZone(); zone != nil { + extra.Add(jsonutils.NewString(zone.Name), "zone") + extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id") + if region := zone.GetRegion(); region != nil { + extra.Add(jsonutils.NewString(region.Name), "region") + } + } + } + guests, guest_status := []string{}, []string{} + for _, guest := range self.GetGuests() { + guests = append(guests, guest.Name) + guest_status = append(guest_status, guest.Status) + } + extra.Add(jsonutils.NewString(strings.Join(guests, ",")), "guest") + extra.Add(jsonutils.NewInt(int64(len(guests))), "guest_count") + extra.Add(jsonutils.NewString(strings.Join(guest_status, ",")), "guest_status") + + if self.PendingDeleted { + pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds)) + extra.Add(jsonutils.NewString(timeutils.FullIsoTime(pendingDeletedAt)), "auto_delete_at") + } + return extra +} + +func (self *SDisk) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SSharableVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + func (self *SDisk) StartDiskDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, isPurge bool) error { params := jsonutils.NewDict() if isPurge { From 122c1d63f298da1dd8800a29e3e83d48b946e021 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sat, 25 Aug 2018 16:02:26 +0800 Subject: [PATCH 34/45] bugfixs for aliyun server create/delete --- pkg/cloudcommon/cronman/cronman.go | 2 ++ pkg/cloudcommon/db/taskman/tasks.go | 31 ++++++++++++----- pkg/compute/guestdrivers/base.go | 14 ++++---- pkg/compute/models/guestnetworks.go | 17 ++++++++++ pkg/compute/models/guests.go | 21 +++++++++--- pkg/compute/tasks/guest_create_task.go | 4 +-- pkg/compute/tasks/guest_delete_task.go | 6 +++- pkg/mcclient/auth/auth.go | 2 +- pkg/util/aliyun/aliyun.go | 1 + pkg/util/aliyun/errors.go | 13 +++++++ pkg/util/aliyun/instance.go | 47 ++++++++++++++++++-------- 11 files changed, 120 insertions(+), 38 deletions(-) create mode 100644 pkg/util/aliyun/errors.go diff --git a/pkg/cloudcommon/cronman/cronman.go b/pkg/cloudcommon/cronman/cronman.go index d01cc096d5..1dba9a33b2 100644 --- a/pkg/cloudcommon/cronman/cronman.go +++ b/pkg/cloudcommon/cronman/cronman.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" ) @@ -89,6 +90,7 @@ func runJob(name string, job func(ctx context.Context, userCred mcclient.TokenCr }() ctx := context.Background() + ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Region-Corn-Service") userCred := auth.AdminCredential() job(ctx, userCred) } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index aa08e2a1e8..3182d47138 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -246,17 +246,21 @@ func (manager *STaskManager) NewParallelTask(ctx context.Context, taskName strin } func (manager *STaskManager) fetchTask(idStr string) *STask { - task, err := db.NewModelObject(manager) + iTask, err := db.NewModelObject(manager) if err != nil { log.Errorf("New task object fail: %s", err) return nil } - err = manager.Query().Equals("id", idStr).First(task) + err = manager.Query().Equals("id", idStr).First(iTask) if err != nil { log.Errorf("GetTask %s fail: %s", idStr, err) return nil } - return task.(*STask) + task := iTask.(*STask) + if task.Params == nil { + task.Params = jsonutils.NewDict() + } + return task } func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) { @@ -285,20 +289,21 @@ func (manager *STaskManager) execTask(taskId string, data jsonutils.JSONObject) } } -func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, isMulti bool) { +func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject, isMulti bool) { var err error ctxData := task.GetRequestContext() ctx := ctxData.GetContext() taskFailed := false + data := odata if data != nil { taskStatus, _ := data.GetString("__status__") if len(taskStatus) > 0 && taskStatus != "OK" { taskFailed = true data, err = data.Get("reason") if err != nil { - data = jsonutils.NewString("Task failed due to unknown remote errors!") + data = jsonutils.NewString(fmt.Sprintf("Task failed due to unknown remote errors! %s", odata)) } } } else { @@ -320,7 +325,13 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, if !funcValue.IsValid() || funcValue.IsNil() { msg := fmt.Sprintf("Stage %s not found", stageName) - log.Errorf(msg) + if taskFailed { + // failed handler is optional, ignore the error + log.Warningf(msg) + msg, _ = data.GetString() + } else { + log.Errorf(msg) + } task.SetStageFailed(ctx, msg) task.SaveRequestContext(&ctxData) return @@ -408,9 +419,11 @@ func (self *STask) GetParentTask() *STask { func (self *STask) GetRequestContext() appctx.AppContextData { ctxData := appctx.AppContextData{} - ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) - if ctxJson != nil { - ctxJson.Unmarshal(&ctxData) + if self.Params != nil { + ctxJson, _ := self.Params.Get(REQUEST_CONTEXT_KEY) + if ctxJson != nil { + ctxJson.Unmarshal(&ctxData) + } } return ctxData } diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 6d10f94aa5..93ab7183cc 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -54,13 +54,13 @@ func (self *SBaseGuestDriver) StartGuestCreateTask(guest *models.SGuest, ctx con } func (self *SBaseGuestDriver) OnGuestCreateTaskComplete(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { - //if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { - // task.SetStage("on_auto_start_guest", nil) - // return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) - //} else { - task.SetStage("on_sync_status_complete", nil) - return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) - //} + if jsonutils.QueryBoolean(task.GetParams(), "auto_start", false) { + task.SetStage("on_auto_start_guest", nil) + return guest.StartGueststartTask(ctx, task.GetUserCred(), nil, task.GetTaskId()) + } else { + task.SetStage("on_sync_status_complete", nil) + return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) + } } func (self *SBaseGuestDriver) StartDeleteGuestTask(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index 3e8a3685ac..514aff944d 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/compute/options" + "database/sql" ) const ( @@ -331,6 +332,22 @@ func (manager *SGuestnetworkManager) DeleteGuestNics(ctx context.Context, guest return nil } +func (manager *SGuestnetworkManager) getGuestNicByIP(ip string) (*SGuestnetwork, error) { + gn := SGuestnetwork{} + q := manager.Query() + q = q.Equals("ip_addr", ip) + err := q.First(&gn) + if err != nil { + if err != sql.ErrNoRows { + log.Errorf("getGuestNicByIP fail %s", err) + return nil, err + } + return nil, nil + } + gn.SetModelManager(manager) + return &gn, nil +} + func (self *SGuestnetwork) LogDetachEvent(userCred mcclient.TokenCredential, guest *SGuest, network *SNetwork) { if network == nil { netTmp, _ := NetworkManager.FetchById(self.NetworkId) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index d47ff0e5f8..7676350d30 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1429,7 +1429,20 @@ func (self *SGuest) SyncVMNics(ctx context.Context, userCred mcclient.TokenCrede if add.net == nil { continue // cannot determine which network it attached to } - err := self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(), + // check if the IP has been occupied, if yes, release the IP + gn, err := GuestnetworkManager.getGuestNicByIP(add.nic.GetIP()) + if err != nil { + result.AddError(err) + continue + } + if gn != nil { + err = gn.Detach(ctx, userCred) + if err != nil { + result.AddError(err) + continue + } + } + err = self.Attach2Network(ctx, userCred, add.net, nil, add.nic.GetIP(), add.nic.GetMAC(), add.nic.GetDriver(), 0, false, -1, add.reserve, IPAllocationDefault, true) if err != nil { result.AddError(err) @@ -2110,9 +2123,9 @@ func (self *SGuest) AllowDeleteItem(ctx context.Context, userCred mcclient.Token func (self *SGuest) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { overridePendingDelete := false purge := false - if data != nil { - overridePendingDelete = jsonutils.QueryBoolean(data, "override_pending_delete", false) - purge = jsonutils.QueryBoolean(data, "purge", false) + if query != nil { + overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false) + purge = jsonutils.QueryBoolean(query, "purge", false) } return self.StartDeleteGuestTask(ctx, userCred, "", purge, overridePendingDelete) } diff --git a/pkg/compute/tasks/guest_create_task.go b/pkg/compute/tasks/guest_create_task.go index 1248fe287e..a57f49afca 100644 --- a/pkg/compute/tasks/guest_create_task.go +++ b/pkg/compute/tasks/guest_create_task.go @@ -105,7 +105,7 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DEPLOY_FAILED, data.String()) } -/*func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { +func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) } @@ -113,4 +113,4 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context func (self *GuestCreateTask) OnSyncStatusComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) self.SetStageComplete(ctx, guest.GetShortDesc()) -}*/ +} diff --git a/pkg/compute/tasks/guest_delete_task.go b/pkg/compute/tasks/guest_delete_task.go index 09666d5e52..7b21109cda 100644 --- a/pkg/compute/tasks/guest_delete_task.go +++ b/pkg/compute/tasks/guest_delete_task.go @@ -4,12 +4,14 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" - "yunion.io/x/pkg/utils" ) type GuestDeleteTask struct { @@ -31,6 +33,7 @@ func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.ISt if options.Options.EnablePendingDelete && !guest.PendingDeleted && !jsonutils.QueryBoolean(self.Params, "purge", false) && !jsonutils.QueryBoolean(self.Params, "override_pending_delete", false) { + log.Debugf("XXXXXXX Do guest pending delete... XXXXXXX") guestStatus, _ := self.Params.GetString("guest_status") if !utils.IsInStringArray(guestStatus, []string{models.VM_SCHEDULE_FAILED, models.VM_NETWORK_FAILED, models.VM_DISK_FAILED, models.VM_CREATE_FAILED, models.VM_DEVICE_FAILED}) { @@ -38,6 +41,7 @@ func (self *GuestDeleteTask) OnGuestStopComplete(ctx context.Context, obj db.ISt return } } + log.Debugf("XXXXXXX Do real delete on guest ... XXXXXXX") self.OnGuestStopCompleteFailed(ctx, guest, data) } diff --git a/pkg/mcclient/auth/auth.go b/pkg/mcclient/auth/auth.go index 214be4b0ad..4e0957e66a 100644 --- a/pkg/mcclient/auth/auth.go +++ b/pkg/mcclient/auth/auth.go @@ -103,7 +103,7 @@ func (c *TokenCacheVerify) Verify(cli *mcclient.Client, adminToken, token string if err != nil { return nil, fmt.Errorf("Add %s credential to cache: %#v", cred.GetTokenString(), err) } - log.Infof("Add token: %s", cred) + // log.Debugf("Add token: %s", cred) return cred, nil } diff --git a/pkg/util/aliyun/aliyun.go b/pkg/util/aliyun/aliyun.go index 26b9efce28..9d002a46d1 100644 --- a/pkg/util/aliyun/aliyun.go +++ b/pkg/util/aliyun/aliyun.go @@ -3,6 +3,7 @@ package aliyun import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" diff --git a/pkg/util/aliyun/errors.go b/pkg/util/aliyun/errors.go new file mode 100644 index 0000000000..da618ab69d --- /dev/null +++ b/pkg/util/aliyun/errors.go @@ -0,0 +1,13 @@ +package aliyun + +import ( + aliyunerrors "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors" +) + +func isError(err error, code string) bool { + aliyunErr, ok := err.(aliyunerrors.Error) + if ! ok { + return false + } + return aliyunErr.ErrorCode() == code +} diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 12cc379e68..2ad1e5f4fc 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -397,7 +397,10 @@ func (self *SRegion) doStopVM(instanceId string, isForce bool) error { } func (self *SRegion) doDeleteVM(instanceId string) error { - return self.instanceOperation(instanceId, "DeleteInstance", nil) + params := make(map[string]string) + params["TerminateSubscription"] = "false" + params["Force"] = "true" + return self.instanceOperation(instanceId, "DeleteInstance", params) } /*func (self *SRegion) waitInstanceStatus(instanceId string, target string, interval time.Duration, timeout time.Duration) error { @@ -420,7 +423,11 @@ func (self *SInstance) waitStatus(target string, interval time.Duration, timeout }*/ func (self *SRegion) StartVM(instanceId string) error { - status, _ := self.GetInstanceStatus(instanceId) + status, err := self.GetInstanceStatus(instanceId) + if err != nil { + log.Errorf("Fail to get instance status on StartVM: %s", err) + return err + } if status != InstanceStatusStopped { log.Errorf("StartVM: vm status is %s expect %s", status, InstanceStatusStopped) return cloudprovider.ErrInvalidStatus @@ -433,7 +440,11 @@ func (self *SRegion) StartVM(instanceId string) error { } func (self *SRegion) StopVM(instanceId string, isForce bool) error { - status, _ := self.GetInstanceStatus(instanceId) + status, err := self.GetInstanceStatus(instanceId) + if err != nil { + log.Errorf("Fail to get instance status on StopVM: %s", err) + return err + } if status != InstanceStatusRunning { log.Errorf("StopVM: vm status is %s expect %s", status, InstanceStatusRunning) return cloudprovider.ErrInvalidStatus @@ -447,14 +458,13 @@ func (self *SRegion) StopVM(instanceId string, isForce bool) error { func (self *SRegion) DeleteVM(instanceId string) error { status, err := self.GetInstanceStatus(instanceId) - if status == InstanceStatusRunning { - err = self.StopVM(instanceId, true) - if err != nil { - return err - } - } else if status != InstanceStatusStopped { - log.Errorf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) - return cloudprovider.ErrInvalidStatus + if err != nil { + log.Errorf("Fail to get instance status on DeleteVM: %s", err) + return err + } + log.Debugf("Instance status on delete is %s", status) + if status != InstanceStatusStopped { + log.Warningf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) } return self.doDeleteVM(instanceId) // if err != nil { @@ -487,9 +497,18 @@ func (self *SInstance) StopVM(isForce bool) error { } func (self *SInstance) DeleteVM() error { - err := self.host.zone.region.DeleteVM(self.InstanceId) - if err != nil { - return err + for { + err := self.host.zone.region.DeleteVM(self.InstanceId) + if err != nil { + if isError(err, "IncorrectInstanceStatus.Initializing") { + log.Infof("The instance is initializing, try later ...") + time.Sleep(10*time.Second) + } else { + return err + } + }else { + break + } } return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes } From c073766ccce803e5938bb32d33561aaffca08df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Sat, 25 Aug 2018 16:49:48 +0800 Subject: [PATCH 35/45] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9B=B4=E6=96=B0secru?= =?UTF-8?q?les=E5=A4=B1=E8=B4=A5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/secgrouprules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 4347ea48c9..46b032e195 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -328,7 +328,7 @@ func (manager *SSecurityGroupRuleManager) SyncRules(ctx context.Context, userCre cmp := strings.Compare(dbStr, ruleStr) if cmp == 0 { if dbRules[j].Description != rules[i].Description { - if _, err := manager.TableSpec().Update(dbRules[j], func() error { + if _, err := manager.TableSpec().Update(&dbRules[j], func() error { dbRules[j].Description = rules[i].Description return nil }); err != nil { From d1607e7be6b7080ca05cc8ef1ae846a2849e70ca Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Sat, 25 Aug 2018 18:58:24 +0800 Subject: [PATCH 36/45] support joint filter --- Gopkg.lock | 15 ++- pkg/cloudcommon/db/db_dispatcher.go | 30 ++++- vendor/yunion.io/x/log/hooks/file.go | 108 ++++++++++++++++++ vendor/yunion.io/x/log/hooks/stdio.go | 7 +- .../x/pkg/util/filterclause/filterclause.go | 36 +++++- .../x/pkg/util/sysutils/storagetypes.go | 8 +- 6 files changed, 190 insertions(+), 14 deletions(-) create mode 100644 vendor/yunion.io/x/log/hooks/file.go diff --git a/Gopkg.lock b/Gopkg.lock index 953506dd3e..dd0d306013 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -666,13 +666,15 @@ version = "v1.2.0" [[projects]] - digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" branch = "master" + digest = "1:c1f242537d2898c84fe9bfd8bd94b177cd5ff98a10c397dbbc494f4e2b173be6" name = "github.com/serialx/hashring" packages = ["."] + pruneopts = "UT" revision = "49a4782e9908fe098c907022a1bd7519c79803d6" [[projects]] + digest = "1:d867dfa6751c8d7a435821ad3b736310c2ed68945d05b50fb9d23aee0540c8cc" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "UT" @@ -1040,18 +1042,18 @@ [[projects]] branch = "master" - digest = "1:b707a5d591b21dad9ebdb264281632ed43d34e2b980dce967acb2c7adcc93ad3" + digest = "1:636db180d3fc734536a5437d93bb6fe937feb036c1ed062c25872a47c4283163" name = "yunion.io/x/log" packages = [ ".", "hooks", ] pruneopts = "UT" - revision = "a10b94c5038480920262e0f344656328434955a2" + revision = "5185c49f6d361f61e561fbccba1bedcc7c62748b" [[projects]] branch = "master" - digest = "1:47670c70d0326f624de21c8f29d0f73850062f0752f27e146308db62fbb540d3" + digest = "1:8e2a4d71738e5e3bffc415b23f100b0a5033dbf88154783122f5147d4fc213b5" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1086,7 +1088,7 @@ "utils", ] pruneopts = "UT" - revision = "6c349ecbdb90d5f6ecc9b86596428176c1937b32" + revision = "7348c84865abac3fcdec618591b7eee7c2e0669b" [[projects]] branch = "master" @@ -1109,6 +1111,7 @@ analyzer-version = 1 input-imports = [ "github.com/aliyun/alibaba-cloud-sdk-go/sdk", + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors", "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests", "github.com/aliyun/aliyun-oss-go-sdk/oss", "github.com/aokoli/goutils", @@ -1147,6 +1150,7 @@ "github.com/mholt/caddy/startupshutdown", "github.com/miekg/dns", "github.com/moul/http2curl", + "github.com/serialx/hashring", "github.com/stretchr/testify/assert", "github.com/vmware/govmomi", "github.com/vmware/govmomi/object", @@ -1158,6 +1162,7 @@ "golang.org/x/crypto/ssh", "gopkg.in/gin-gonic/gin.v1", "k8s.io/api/core/v1", + "k8s.io/apimachinery/pkg/api/errors", "k8s.io/apimachinery/pkg/apis/meta/v1", "k8s.io/apimachinery/pkg/fields", "k8s.io/apimachinery/pkg/labels", diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 97df816bde..57b6ea13c9 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -251,6 +251,30 @@ func applyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery, return q, nil } +func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, jointFilters []string, filterAny bool) (*sqlchemy.SQuery, error) { + for _, f := range jointFilters { + jfc := filterclause.ParseJointFilterClause(f) + if jfc != nil { + jointModelManager := GetModelManager(jfc.GetJointModelName()) + schFields := searchFields(jointModelManager, userCred) + if ok, _ := utils.InStringArray(jfc.GetField(), schFields); ok { + sq := jointModelManager.Query(jfc.ReleatedKey) + cond := jfc.GetJointFilter(sq) + if cond != nil { + sq = sq.Filter(cond) + if filterAny { + q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("id"), sq))) + } else { + q = q.Filter(sqlchemy.AND(sqlchemy.In(q.Field("id"), sq))) + } + } + } + } + } + return q, nil +} + func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { @@ -272,11 +296,15 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem return nil, err } } + filterAny, _ := query.Bool("filter_any") filters := jsonutils.GetQueryStringArray(query, "filter") if len(filters) > 0 { - filterAny, _ := query.Bool("filter_any") q, err = applyListItemsGeneralFilters(manager, q, userCred, filters, filterAny) } + jointFilter := jsonutils.GetQueryStringArray(query, "joint_filter") + if len(jointFilter) > 0 { + q, _ = applyListItemsGeneralJointFilters(manager, q, userCred, jointFilter, filterAny) + } return q, nil } diff --git a/vendor/yunion.io/x/log/hooks/file.go b/vendor/yunion.io/x/log/hooks/file.go new file mode 100644 index 0000000000..3a99b0397a --- /dev/null +++ b/vendor/yunion.io/x/log/hooks/file.go @@ -0,0 +1,108 @@ +package hooks + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/sirupsen/logrus" +) + +type LogFileHook struct { + FileDir string + FileName string + fullPath string + file *os.File + written int64 +} + +func (h *LogFileHook) Init() error { + if fi, err := os.Lstat(h.FileDir); err != nil { + if os.IsNotExist(err) { + os.MkdirAll(h.FileDir, 0755) + } else { + return fmt.Errorf("Lstat %s: %s", h.FileDir, err) + } + } else if !fi.Mode().IsDir() { + return fmt.Errorf("%s exists and it's not a directory", h.FileDir) + } + + h.fullPath = filepath.Join(h.FileDir, h.FileName) + file, err := os.OpenFile(h.fullPath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) + if err != nil { + return fmt.Errorf("OpenFile %s: %s", h.fullPath, err) + } + h.file = file + h.written = 0 + return nil +} + +func (h *LogFileHook) DeInit() { + if h.file != nil { + h.file.Close() + } +} + +func (h *LogFileHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *LogFileHook) Fire(e *logrus.Entry) error { + if b, err := e.Logger.Formatter.Format(e); err != nil { + return err + } else { + n, err := h.file.Write(b) + h.written += int64(n) + return err + } + return nil +} + +func (h *LogFileHook) Written() int64 { + return h.written +} + +// rotate by size +type LogFileRotateHook struct { + LogFileHook + RotateNum int + RotateSize int64 + filePaths []string +} + +func (h *LogFileRotateHook) Init() error { + if err := h.LogFileHook.Init(); err != nil { + return err + } + h.filePaths = make([]string, h.RotateNum) + for i := 1; i < h.RotateNum; i++ { + fileName := fmt.Sprintf("%s.%d", h.FileName, i) + filePath := filepath.Join(h.FileDir, fileName) + h.filePaths[i] = filePath + } + h.filePaths[0] = filepath.Join(h.FileDir, h.FileName) + return nil +} + +func (h *LogFileRotateHook) rotate() { + for i := h.RotateNum - 1; i > 0; i-- { + filePath0 := h.filePaths[i-1] + if _, err := os.Lstat(filePath0); err != nil { + continue + } + filePath1 := h.filePaths[i] + os.Rename(filePath0, filePath1) + } + h.LogFileHook.DeInit() + h.LogFileHook.Init() +} + +func (h *LogFileRotateHook) Fire(e *logrus.Entry) error { + if err := h.LogFileHook.Fire(e); err != nil { + return err + } + if h.LogFileHook.Written() >= h.RotateSize { + h.rotate() + } + return nil +} diff --git a/vendor/yunion.io/x/log/hooks/stdio.go b/vendor/yunion.io/x/log/hooks/stdio.go index 6fa3dc4381..e5d32000ad 100644 --- a/vendor/yunion.io/x/log/hooks/stdio.go +++ b/vendor/yunion.io/x/log/hooks/stdio.go @@ -12,14 +12,9 @@ type StdioHook struct{} func (hook *StdioHook) Fire(entry *logrus.Entry) error { line, err := entry.String() if err != nil { - fmt.Fprintf(os.Stderr, "Unable to read entry, %v", err) return err } - if entry.Level >= logrus.ErrorLevel { - fmt.Fprintf(os.Stderr, line) - } else { - fmt.Fprintf(os.Stdout, line) - } + fmt.Fprintf(os.Stderr, line) return nil } diff --git a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go index 2a3a1ccd10..5518ea0f3b 100644 --- a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go +++ b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go @@ -16,6 +16,20 @@ type SFilterClause struct { params []string } +type SJointFilterClause struct { + SFilterClause + JointModel string + ReleatedKey string +} + +func (jfc *SJointFilterClause) GetJointFilter(q *sqlchemy.SQuery) sqlchemy.ICondition { + return jfc.QueryCondition(q) +} + +func (jfc *SJointFilterClause) GetJointModelName() string { + return jfc.JointModel[:len(jfc.JointModel)-1] +} + func (fc *SFilterClause) QueryCondition(q *sqlchemy.SQuery) sqlchemy.ICondition { field := q.Field(fc.field) if field == nil { @@ -63,11 +77,13 @@ func (fc *SFilterClause) String() string { } var ( - filterClausePattern *regexp.Regexp + filterClausePattern *regexp.Regexp + jointFilterClausePattern *regexp.Regexp ) func init() { filterClausePattern = regexp.MustCompile(`^(\w+)\.(\w+)\((.*)\)`) + jointFilterClausePattern = regexp.MustCompile(`^(\w+)\((\w+)\).(\w+)\.(\w+)\((.*)\)`) } func ParseFilterClause(filter string) *SFilterClause { @@ -79,3 +95,21 @@ func ParseFilterClause(filter string) *SFilterClause { fc := SFilterClause{field: matches[1], funcName: matches[2], params: params} return &fc } + +func ParseJointFilterClause(jointFilter string) *SJointFilterClause { + matches := jointFilterClausePattern.FindStringSubmatch(jointFilter) + if matches == nil { + return nil + } + params := utils.FindWords([]byte(matches[5]), 0) + jfc := SJointFilterClause{ + SFilterClause: SFilterClause{ + field: matches[3], + funcName: matches[4], + params: params, + }, + JointModel: matches[1], + ReleatedKey: matches[2], + } + return &jfc +} diff --git a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go index 476aa53bff..16baa403d2 100644 --- a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go +++ b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go @@ -9,12 +9,18 @@ const ( STORAGE_NAS = "nas" STORAGE_VSAN = "vsan" + STORAGE_CLOUD = "cloud" STORAGE_CLOUD_SSD = "cloud_ssd" STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" + + STORAGE_STANDARD = "standard" //Azure hdd storage type + STORAGE_PREMIUM = "premium" //Azure ssd storage type ) var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_CLOUD_SSD, STORAGE_CLOUD_EFFICIENCY} + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, + STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_EFFICIENCY, + STORAGE_STANDARD, STORAGE_PREMIUM} var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} From e6b77164c4e830737b266ded205cdefedc981105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 27 Aug 2018 11:55:22 +0800 Subject: [PATCH 37/45] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E5=9B=9E=E6=94=B6=E7=AB=99=E6=97=B6=E6=9C=AA=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E8=AE=B0=E5=BD=95bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/hostdrivers/aliyun.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 0bcfa18005..57590014f8 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -81,9 +81,13 @@ func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, s return err } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { return err + } else if err := iDisk.Delete(); err != nil { + return err } else { - return iDisk.Delete() + data := jsonutils.NewDict() + task.ScheduleRun(data) } + return nil } func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { From 02e16544187f50939188b05bf84027414fa12806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 27 Aug 2018 12:00:59 +0800 Subject: [PATCH 38/45] =?UTF-8?q?=E6=99=AE=E9=80=9A=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E4=BA=91=E7=9B=98=E6=97=B6=EF=BC=8C=E9=9C=80?= =?UTF-8?q?=E8=A6=81storage-list=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/storages.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 28650d29f1..3797e8a33e 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -97,6 +97,10 @@ func (self *SStorage) IsLocal() bool { return self.StorageType == STORAGE_LOCAL || self.StorageType == STORAGE_BAREMETAL } +func (manager *SStorageManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return true +} + func (self *SStorage) GetUsedCapacity(isReady tristate.TriState) int { disks := DiskManager.Query().SubQuery() q := disks.Query(sqlchemy.SUM("sum", disks.Field("disk_size"))).Equals("storage_id", self.Id) From 852ba299e3e9ca74ae557450c36a1a2e2d3fc238 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Mon, 27 Aug 2018 12:15:21 +0800 Subject: [PATCH 39/45] region-dns: add vars to build rpm package --- build/region-dns/vars | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 build/region-dns/vars diff --git a/build/region-dns/vars b/build/region-dns/vars new file mode 100644 index 0000000000..778d49a5da --- /dev/null +++ b/build/region-dns/vars @@ -0,0 +1,2 @@ +DESCRIPTION="Yunion Cloud Region DNS Service" +# SERVICE="yes" From 23ebca6cfe41f016c2ea309a20a1246622ef7a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 27 Aug 2018 12:18:18 +0800 Subject: [PATCH 40/45] =?UTF-8?q?=E6=B7=BB=E5=8A=A0interface=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudprovider/resources.go | 3 +- pkg/compute/guestdrivers/aliyun.go | 50 ++++++++++++++++++++----- pkg/compute/hostdrivers/aliyun.go | 16 +++++++- pkg/compute/hostdrivers/kvm.go | 2 +- pkg/compute/models/disks.go | 54 ++++++++++++++++++++++----- pkg/compute/models/guests.go | 35 ++++++++++++++++- pkg/compute/models/hostdrivers.go | 2 +- pkg/compute/tasks/disk_create_task.go | 4 +- pkg/util/aliyun/disk.go | 10 +++++ pkg/util/aliyun/host.go | 4 ++ pkg/util/aliyun/image.go | 4 ++ pkg/util/aliyun/instance.go | 15 ++++++++ pkg/util/aliyun/region.go | 4 ++ pkg/util/aliyun/securitygroup.go | 4 ++ pkg/util/aliyun/storage.go | 4 ++ pkg/util/aliyun/storagecache.go | 5 +++ pkg/util/aliyun/vpc.go | 4 ++ pkg/util/aliyun/vswitch.go | 4 ++ pkg/util/aliyun/wire.go | 5 +++ pkg/util/aliyun/zone.go | 5 +++ pkg/util/esxi/host.go | 4 ++ pkg/util/esxi/storage.go | 6 ++- pkg/util/esxi/virtualmachine.go | 8 +++- 23 files changed, 221 insertions(+), 31 deletions(-) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index ec124341c0..3c333f8946 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -18,6 +18,7 @@ type ICloudResource interface { Refresh() error IsEmulated() bool + GetMetadata() *jsonutils.JSONDict } type ICloudRegion interface { @@ -156,7 +157,7 @@ type ICloudVM interface { UpdateVM(name string) error RebuildRoot(imageId string) error DeployVM(name string, password string, publicKey string, resetPassword bool, deleteKeypair bool, description string) error - ChangeConfig(instanceId string,ncpu int, vmem int) error + ChangeConfig(instanceId string, ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) AttachDisk(diskId string) error } diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index c8953351ad..45226734d3 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/httperrors" @@ -116,8 +117,9 @@ func (self *SAliyunGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *mo } type SDiskInfo struct { - Size int - Uuid string + Size int + Uuid string + Metadata map[string]string } func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { @@ -207,12 +209,19 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu dinfo := SDiskInfo{} dinfo.Uuid = idisks[i].GetGlobalId() dinfo.Size = idisks[i].GetDiskSizeMB() + if metaData := idisks[i].GetMetadata(); metaData != nil { + dinfo.Metadata = make(map[string]string, 0) + if err := metaData.Unmarshal(dinfo.Metadata); err != nil { + log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err) + } + } diskInfo[i] = dinfo } data.Add(jsonutils.Marshal(&diskInfo), "disks") } data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + data.Add(iVM.GetMetadata(), "metadata") return data, nil }) @@ -236,7 +245,7 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu resetPassword := jsonutils.QueryBoolean(params, "reset_password", false) deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) password, _ := params.GetString("password") - if resetPassword && len(password) == 0{ + if resetPassword && len(password) == 0 { password = seclib2.RandomPassword2(12) } @@ -245,15 +254,15 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu publicKey = k } - taskman.LocalTaskRun(task, func () (jsonutils.JSONObject, error) { + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { encpasswd, err := utils.EncryptAESBase64(guest.Id, password) if err != nil { log.Errorf("encrypt password failed %s", err) } data := jsonutils.NewDict() - data.Add(jsonutils.NewString("root"), "account") // 用户名 - data.Add(jsonutils.NewString(encpasswd), "key") // 密码 + data.Add(jsonutils.NewString("root"), "account") // 用户名 + data.Add(jsonutils.NewString(encpasswd), "key") // 密码 e := iVM.DeployVM(name, password, publicKey, resetPassword, deleteKeypair, description) return data, e }) @@ -284,6 +293,13 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex disk.DiskSize = diskInfo[i].Size disk.ExternalId = diskInfo[i].Uuid disk.Status = models.DISK_READY + if len(diskInfo[i].Metadata) > 0 { + for key, value := range diskInfo[i].Metadata { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } return nil }) if err != nil { @@ -299,6 +315,20 @@ func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Contex if len(uuid) > 0 { guest.SetExternalId(uuid) } + + if metaData, _ := data.Get("metadata"); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get guest %s metadata error: %v", guest.Name, err) + } else { + for key, value := range meta { + if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) + } + } + } + } + guest.SaveDeployInfo(ctx, task.GetUserCred(), data) return nil } @@ -320,9 +350,9 @@ func (self *SAliyunGuestDriver) RequestSyncConfigOnHost(ctx context.Context, gue } type SAliyunVMChangeConfig struct { - InstanceId string - Cpu int - Memory int + InstanceId string + Cpu int + Memory int } func (self *SAliyunGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { @@ -432,4 +462,4 @@ func (self *SAliyunGuestDriver) RequestRebuildRootDisk(ctx context.Context, gues task.ScheduleRun(nil) return nil -} \ No newline at end of file +} diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 0bcfa18005..b96ea60a30 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -48,7 +48,7 @@ func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host * return nil } -func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { +func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { if iCloudStorage, err := storage.GetIStorage(); err != nil { return err } else { @@ -61,6 +61,20 @@ func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, } else { if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { disk.ExternalId = iDisk.GetGlobalId() + + if metaData := iDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) + } else { + for key, value := range meta { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + } + return nil }); err != nil { log.Errorf("Update disk externalId err: %v", err) diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 0a5cbca5b2..ed08542a36 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -74,7 +74,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod return nil } -func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { +func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { header := http.Header{} header.Add("X-Task-Id", task.GetTaskId()) header.Add("X-Region-Version", "v2") diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 3081709834..b05fa85b5e 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -297,7 +297,7 @@ func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.To return nil } -func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { +func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SStorage, taskId string, userCred mcclient.TokenCredential, rebuild bool, snapshot string, task taskman.ITask) error { log.Infof("Allocating disk on host %s ...", host.GetName()) templateId := self.GetTemplateId() @@ -326,7 +326,7 @@ func (self *SDisk) StartAllocate(host *SHost, storage *SStorage, taskId string, if rebuild { content.Add(jsonutils.JSONTrue, "rebuild") } - return host.GetHostDriver().RequestAllocateDiskOnStorage(host, storage, self, task, content) + return host.GetHostDriver().RequestAllocateDiskOnStorage(ctx, host, storage, self, task, content) } func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { @@ -433,7 +433,7 @@ func (manager *SDiskManager) getDisksByStorage(storage *SStorage) ([]SDisk, erro return disks, nil } -func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) { +func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, vdisk cloudprovider.ICloudDisk) (*SDisk, error) { diskObj, err := manager.FetchByExternalId(vdisk.GetGlobalId()) if err != nil { if err == sql.ErrNoRows { @@ -444,13 +444,13 @@ func (manager *SDiskManager) syncCloudDisk(userCred mcclient.TokenCredential, vd return nil, err } storage := storageObj.(*SStorage) - return manager.newFromCloudDisk(userCred, vdisk, storage) + return manager.newFromCloudDisk(ctx, userCred, vdisk, storage) } else { return nil, err } } else { disk := diskObj.(*SDisk) - err = disk.syncWithCloudDisk(userCred, vdisk) + err = disk.syncWithCloudDisk(ctx, userCred, vdisk) if err != nil { return nil, err } @@ -490,7 +490,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudDisk(userCred, commonext[i]) + err = commondb[i].syncWithCloudDisk(ctx, userCred, commonext[i]) if err != nil { syncResult.UpdateError(err) } else { @@ -501,7 +501,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To } for i := 0; i < len(added); i += 1 { - new, err := manager.newFromCloudDisk(userCred, added[i], storage) + new, err := manager.newFromCloudDisk(ctx, userCred, added[i], storage) if err != nil { syncResult.AddError(err) } else { @@ -514,7 +514,7 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To return localDisks, remoteDisks, syncResult } -func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { +func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { extDisk.Refresh() self.Name = extDisk.GetName() @@ -535,11 +535,26 @@ func (self *SDisk) syncWithCloudDisk(userCred mcclient.TokenCredential, extDisk }) if err != nil { log.Errorf("syncWithCloudDisk error %s", err) + return err } - return err + + if metaData := extDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := self.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", self.Name, key, value, err) + } + } + } + } + + return nil } -func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) { +func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mcclient.TokenCredential, extDisk cloudprovider.ICloudDisk, storage *SStorage) (*SDisk, error) { disk := SDisk{} disk.SetModelManager(manager) @@ -562,6 +577,20 @@ func (manager *SDiskManager) newFromCloudDisk(userCred mcclient.TokenCredential, log.Errorf("newFromCloudZone fail %s", err) return nil, err } + + if metaData := extDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := disk.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + } + return &disk, nil } @@ -891,6 +920,11 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict { storage := self.GetStorage() desc.Add(jsonutils.NewString(storage.StorageType), "storage_type") desc.Add(jsonutils.NewString(storage.MediumType), "medium_type") + + if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { + desc.Add(jsonutils.NewString(priceKey), "price_key") + } + fs := self.GetFsFormat() if len(fs) > 0 { desc.Add(jsonutils.NewString(fs), "fs_format") diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 977a5dc28c..d4251224d2 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1249,6 +1249,18 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token db.OpsLog.LogEvent(self, db.ACT_UPDATE, diffStr, userCred) } } + if metaData := extVM.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := self.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", self.Name, key, value, err) + } + } + } + } return nil } @@ -1279,6 +1291,20 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. if err != nil { log.Errorf("Insert fail %s", err) } + + if metaData := extVM.GetMetadata(); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get VM Metadata error: %v", err) + } else { + for key, value := range meta { + if err := guest.SetMetadata(ctx, key, value, userCred); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) + } + } + } + } + return &guest, nil } @@ -1668,7 +1694,7 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred if len(vdisks[i].GetGlobalId()) == 0 { continue } - disk, err := DiskManager.syncCloudDisk(userCred, vdisks[i]) + disk, err := DiskManager.syncCloudDisk(ctx, userCred, vdisks[i]) if err != nil { result.Error(err) return result @@ -3140,6 +3166,11 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { if intBw > 0 { desc.Set("int_bandwidth", jsonutils.NewInt(int64(intBw))) } + + if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { + desc.Add(jsonutils.NewString(priceKey), "price_key") + } + desc.Set("hypervisor", jsonutils.NewString(self.GetHypervisor())) spec := self.GetSpec(false) if self.GetHypervisor() == HYPERVISOR_BAREMETAL { @@ -3454,4 +3485,4 @@ func (manager *SGuestManager) CleanPendingDeleteServers(ctx context.Context, use for i := 0; i < len(guests); i += 1 { guests[i].StartDeleteGuestTask(ctx, userCred, "", false, true) } -} \ No newline at end of file +} diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go index 3f162d75da..6749e5ac68 100644 --- a/pkg/compute/models/hostdrivers.go +++ b/pkg/compute/models/hostdrivers.go @@ -12,7 +12,7 @@ import ( type IHostDriver interface { GetHostType() string CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, scimg *SStoragecachedimage, task taskman.ITask) error - RequestAllocateDiskOnStorage(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error + RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error diff --git a/pkg/compute/tasks/disk_create_task.go b/pkg/compute/tasks/disk_create_task.go index 1c47d60d5a..be0e2635ed 100644 --- a/pkg/compute/tasks/disk_create_task.go +++ b/pkg/compute/tasks/disk_create_task.go @@ -40,10 +40,10 @@ func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, dis } storage := disk.GetStorage() host := storage.GetMasterHost() - db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.GetUserCred()) + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATING, disk.GetShortDesc(), self.GetUserCred()) disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "") self.SetStage("on_disk_ready", nil) - if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { + if err := disk.StartAllocate(ctx, host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error())) } } diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index ca6f21d4f8..7349b8c7c0 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -54,6 +54,16 @@ type SDisk struct { ZoneId string } +func (self *SDisk) GetMetadata() *jsonutils.JSONDict { + data := jsonutils.NewDict() + + // The pricingInfo key structure is 'RegionId::DiskCategory::DiskType + priceKey := fmt.Sprintf("%s::%s::%s", self.RegionId, self.Category, self.Type) + data.Add(jsonutils.NewString(priceKey), "price_key") + + return data +} + func (self *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, offset int, limit int) ([]SDisk, int, error) { if limit > 50 || limit <= 0 { limit = 50 diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index cf842879ad..24c5a2a1b1 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -19,6 +19,10 @@ type SHost struct { zone *SZone } +func (self *SHost) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { return self.zone.GetIWires() } diff --git a/pkg/util/aliyun/image.go b/pkg/util/aliyun/image.go index 9dfb6f0b3f..849e9eb570 100644 --- a/pkg/util/aliyun/image.go +++ b/pkg/util/aliyun/image.go @@ -55,6 +55,10 @@ type SImage struct { Usage string } +func (self *SImage) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SImage) GetId() string { return self.ImageId } diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 12874bc1bb..7f034089b6 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" "time" + "yunion.io/x/onecloud/pkg/util/seclib2" "yunion.io/x/jsonutils" @@ -164,6 +165,20 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit return instances, int(total), nil } +func (self *SInstance) GetMetadata() *jsonutils.JSONDict { + data := jsonutils.NewDict() + + // The pricingInfo key structure is 'RegionId::InstanceType::NetworkType::OSType::IoOptimized' + optimized := "optimized" + if !self.IoOptimized { + optimized = "none" + } + priceKey := fmt.Sprintf("%s::%s::%s::%s::%s", self.RegionId, self.InstanceType, self.InstanceNetworkType, self.OSType, optimized) + data.Add(jsonutils.NewString(priceKey), "price_key") + + return data +} + func (self *SInstance) GetCreateTime() time.Time { return self.CreationTime } diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index 78efb6a653..311f7bf195 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -36,6 +36,10 @@ func (self *SRegion) GetClient() *SAliyunClient { return self.client } +func (self *SRegion) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SRegion) getEcsClient() (*sdk.Client, error) { if self.ecsClient == nil { cli, err := sdk.NewClientWithAccessKey(self.RegionId, self.client.accessKey, self.client.secret) diff --git a/pkg/util/aliyun/securitygroup.go b/pkg/util/aliyun/securitygroup.go index 04d787d838..3082451737 100644 --- a/pkg/util/aliyun/securitygroup.go +++ b/pkg/util/aliyun/securitygroup.go @@ -77,6 +77,10 @@ func (v PermissionSet) Less(i, j int) bool { return false } +func (self *SSecurityGroup) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SSecurityGroup) GetId() string { return self.SecurityGroupId } diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index 8f66d8b68b..d8dec31739 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -15,6 +15,10 @@ type SStorage struct { storageType string } +func (self *SStorage) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SStorage) GetId() string { return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) } diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go index 61394948c2..fab2072253 100644 --- a/pkg/util/aliyun/storagecache.go +++ b/pkg/util/aliyun/storagecache.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -19,6 +20,10 @@ type SStoragecache struct { iimages []cloudprovider.ICloudImage } +func (self *SStoragecache) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SStoragecache) GetId() string { return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) } diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index 6fb04fcc0d..ea645ff2be 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -45,6 +45,10 @@ type SVpc struct { VpcName string } +func (self *SVpc) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVpc) GetId() string { return self.VpcId } diff --git a/pkg/util/aliyun/vswitch.go b/pkg/util/aliyun/vswitch.go index 9c9e7bbe84..bef2c3d74d 100644 --- a/pkg/util/aliyun/vswitch.go +++ b/pkg/util/aliyun/vswitch.go @@ -34,6 +34,10 @@ type SVSwitch struct { ZoneId string } +func (self *SVSwitch) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVSwitch) GetId() string { return self.VSwitchId } diff --git a/pkg/util/aliyun/wire.go b/pkg/util/aliyun/wire.go index 903dcc681a..12793359c1 100644 --- a/pkg/util/aliyun/wire.go +++ b/pkg/util/aliyun/wire.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -14,6 +15,10 @@ type SWire struct { inetworks []cloudprovider.ICloudNetwork } +func (self *SWire) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SWire) GetId() string { return fmt.Sprintf("%s-%s", self.vpc.GetId(), self.zone.GetId()) } diff --git a/pkg/util/aliyun/zone.go b/pkg/util/aliyun/zone.go index 844b4d6e6f..2241e4d9ce 100644 --- a/pkg/util/aliyun/zone.go +++ b/pkg/util/aliyun/zone.go @@ -3,6 +3,7 @@ package aliyun import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -111,6 +112,10 @@ type SZone struct { AvailableDedicatedHostTypes SDedicatedHostTypes } +func (self *SZone) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SZone) GetId() string { return self.ZoneId } diff --git a/pkg/util/esxi/host.go b/pkg/util/esxi/host.go index 4f524948d2..3a9a053c5d 100644 --- a/pkg/util/esxi/host.go +++ b/pkg/util/esxi/host.go @@ -68,6 +68,10 @@ func NewHost(manager *SESXiClient, host *mo.HostSystem, dc *SDatacenter) *SHost return &SHost{SManagedObject: newManagedObject(manager, host, dc)} } +func (self *SHost) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SHost) getHostSystem() *mo.HostSystem { return self.object.(*mo.HostSystem) } diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index dc154261d2..b64c57ce1f 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -18,6 +18,10 @@ func NewDatastore(manager *SESXiClient, ds *mo.Datastore, dc *SDatacenter) *SDat return &SDatastore{SManagedObject: newManagedObject(manager, ds, dc)} } +func (self *SDatastore) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SDatastore) getDatastore() *mo.Datastore { return self.object.(*mo.Datastore) } @@ -89,4 +93,4 @@ func (self *SDatastore) GetManagerId() string { func (self *SDatastore) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented -} \ No newline at end of file +} diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 75ccccc448..34c1a9c682 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -27,6 +27,10 @@ func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacen return &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc), host: host} } +func (self *SVirtualMachine) GetMetadata() *jsonutils.JSONDict { + return nil +} + func (self *SVirtualMachine) getVirtualMachine() *mo.VirtualMachine { return self.object.(*mo.VirtualMachine) } @@ -226,6 +230,6 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { return ret, nil } -func (dc *SVirtualMachine) ChangeConfig(instanceId string,ncpu int, vmem int) error { +func (dc *SVirtualMachine) ChangeConfig(instanceId string, ncpu int, vmem int) error { return cloudprovider.ErrNotImplemented -} \ No newline at end of file +} From 47abe37aa7201ccda992a558d0d64872f382305e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 28 Aug 2018 11:09:51 +0800 Subject: [PATCH 41/45] =?UTF-8?q?=E8=99=9A=E6=9C=BA=E5=85=B3=E6=9C=BA?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=97=B6=E6=90=BA=E5=B8=A6note=E4=BF=A1?= =?UTF-8?q?=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/tasks/guest_stop_task.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/tasks/guest_stop_task.go b/pkg/compute/tasks/guest_stop_task.go index 2ed0f4aa6c..e8c3e8c1e6 100644 --- a/pkg/compute/tasks/guest_stop_task.go +++ b/pkg/compute/tasks/guest_stop_task.go @@ -52,7 +52,7 @@ func (self *GuestStopTask) OnGuestStopTaskComplete(ctx context.Context, obj db.I if !self.isSubtask() { guest.SetStatus(self.UserCred, models.VM_READY, "") } - db.OpsLog.LogEvent(guest, db.ACT_STOP, nil, self.UserCred) + db.OpsLog.LogEvent(guest, db.ACT_STOP, guest.GetShortDesc(), self.UserCred) models.HostManager.ClearSchedDescCache(guest.HostId) self.SetStageComplete(ctx, nil) if guest.Status == models.VM_READY && guest.DisableDelete.IsFalse() && guest.ShutdownBehavior == models.SHUTDOWN_TERMINATE { From 1e5a231e0686acf0006ba330bae1bc38da95d3ff Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Tue, 28 Aug 2018 13:39:54 +0800 Subject: [PATCH 42/45] make dep and fix words releated->related --- Gopkg.lock | 4 ++-- pkg/cloudcommon/db/db_dispatcher.go | 2 +- vendor/yunion.io/x/pkg/gotypes/gotypes.go | 20 ++++++++++++++++++- .../x/pkg/util/filterclause/filterclause.go | 8 ++++---- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index dd0d306013..99299205d0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1053,7 +1053,7 @@ [[projects]] branch = "master" - digest = "1:8e2a4d71738e5e3bffc415b23f100b0a5033dbf88154783122f5147d4fc213b5" + digest = "1:8a2992edca980cb03eb9dc89f62c03c8ab6fe0c5ca4fc1da71eb736aae635f65" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1088,7 +1088,7 @@ "utils", ] pruneopts = "UT" - revision = "7348c84865abac3fcdec618591b7eee7c2e0669b" + revision = "1e97a2736e5b90c0274b7d391b8d97aa9107725c" [[projects]] branch = "master" diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 57b6ea13c9..6c8c336a87 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -259,7 +259,7 @@ func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery jointModelManager := GetModelManager(jfc.GetJointModelName()) schFields := searchFields(jointModelManager, userCred) if ok, _ := utils.InStringArray(jfc.GetField(), schFields); ok { - sq := jointModelManager.Query(jfc.ReleatedKey) + sq := jointModelManager.Query(jfc.RelatedKey) cond := jfc.GetJointFilter(sq) if cond != nil { sq = sq.Filter(cond) diff --git a/vendor/yunion.io/x/pkg/gotypes/gotypes.go b/vendor/yunion.io/x/pkg/gotypes/gotypes.go index e589aa7c61..615499fa4a 100644 --- a/vendor/yunion.io/x/pkg/gotypes/gotypes.go +++ b/vendor/yunion.io/x/pkg/gotypes/gotypes.go @@ -120,6 +120,15 @@ func ParseValue(val string, tp reflect.Type) (reflect.Value, error) { } case reflect.String: return reflect.ValueOf(val), nil + case reflect.Ptr: + tpElem := tp.Elem() + rv, err := ParseValue(val, tpElem) + if err != nil { + return reflect.ValueOf(val), fmt.Errorf("Cannot parse %s to %s", val, tp) + } + rvv := reflect.New(tpElem) + rvv.Elem().Set(rv) + return rvv, nil default: if tp == TimeType { tm, e := timeutils.ParseTimeStr(val) @@ -173,7 +182,16 @@ func SetValue(value reflect.Value, valStr string) error { Float32SliceType, Float64SliceType, StringSliceType: reflect.Append(value, reflect.ValueOf(valStr)) default: - return fmt.Errorf("Unsupported type: %v", value.Type()) + if value.Kind() == reflect.Ptr && value.Elem().Kind() != reflect.Slice { + newVal := reflect.New(value.Type().Elem()) + newValElem := newVal.Elem() + if err := SetValue(newValElem, valStr); err != nil { + return err + } + value.Set(newVal) + } else { + return fmt.Errorf("Unsupported type: %v", value.Type()) + } } return nil } diff --git a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go index 5518ea0f3b..a0928f6527 100644 --- a/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go +++ b/vendor/yunion.io/x/pkg/util/filterclause/filterclause.go @@ -18,8 +18,8 @@ type SFilterClause struct { type SJointFilterClause struct { SFilterClause - JointModel string - ReleatedKey string + JointModel string + RelatedKey string } func (jfc *SJointFilterClause) GetJointFilter(q *sqlchemy.SQuery) sqlchemy.ICondition { @@ -108,8 +108,8 @@ func ParseJointFilterClause(jointFilter string) *SJointFilterClause { funcName: matches[4], params: params, }, - JointModel: matches[1], - ReleatedKey: matches[2], + JointModel: matches[1], + RelatedKey: matches[2], } return &jfc } From bf7c07174bd0e662002bf43ec3ff7f74a133a9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 28 Aug 2018 11:25:25 +0800 Subject: [PATCH 43/45] =?UTF-8?q?=E9=98=BF=E9=87=8C=E4=BA=91=E6=9C=BA?= =?UTF-8?q?=E5=99=A8=E5=90=AF=E5=8A=A8=E5=89=8D=E6=A3=80=E6=9F=A5=E7=8A=B6?= =?UTF-8?q?=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/aliyun.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 45226734d3..f149f322ec 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -410,13 +410,17 @@ func (self *SAliyunGuestDriver) RequestStartOnHost(ctx context.Context, guest *m return nil, e } - err := ivm.StartVM() - if err != nil { - return nil, e + result := jsonutils.NewDict() + if ivm.GetStatus() != models.VM_RUNNING { + if err := ivm.StartVM(); err != nil { + return nil, e + } else { + task.ScheduleRun(result) + } + } else { + result.Add(jsonutils.NewBool(true), "is_running") } - result := jsonutils.NewDict() - result.Add(jsonutils.NewBool(true), "is_running") return result, e } From d52438c1e66180de9092fb7826734ab2573f6760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 28 Aug 2018 19:58:50 +0800 Subject: [PATCH 44/45] =?UTF-8?q?=E4=B8=8D=E5=85=81=E8=AE=B8=E5=8D=B8?= =?UTF-8?q?=E8=BD=BD=E7=B3=BB=E7=BB=9F=E7=9B=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guests.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index d4251224d2..afd50582dc 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2369,6 +2369,9 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok disk := iDisk.(*SDisk) if disk != nil { if self.isAttach2Disk(disk) { + if disk.DiskType == DISK_TYPE_SYS { + return nil, httperrors.NewUnsupportOperationError("Cannot detach sys disk") + } detachDiskStatus, err := self.GetDriver().GetDetachDiskStatus() if err != nil { return nil, err From 876f4ce1fadb9b49f5d54c6f210213221879861b Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 29 Aug 2018 11:21:21 +0800 Subject: [PATCH 45/45] climc: server-rebuild-root support keypair and passwd --- cmd/climc/shell/keypairs.go | 4 ++-- cmd/climc/shell/servers.go | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cmd/climc/shell/keypairs.go b/cmd/climc/shell/keypairs.go index a13ae163a7..d56d2122a2 100644 --- a/cmd/climc/shell/keypairs.go +++ b/cmd/climc/shell/keypairs.go @@ -113,9 +113,9 @@ func init() { if len(args.PublicKey) > 0 { content, e := ioutil.ReadFile(args.PublicKey) if e != nil { - params.Add(jsonutils.NewString(args.PublicKey), "public_Key") + params.Add(jsonutils.NewString(args.PublicKey), "public_key") } else { - params.Add(jsonutils.NewString(string(content)), "public_Key") + params.Add(jsonutils.NewString(string(content)), "public_key") } } else { return fmt.Errorf("no public key provided") diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index f0d74b5d59..09e610c86f 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -644,14 +644,26 @@ func init() { }) type ServerRebuildRootOptions struct { - SERVER string `help:"Server to rebuild root"` - Image string `help:"New root Image template ID"` + SERVER string `help:"Server to rebuild root"` + AutoStart bool `help:"Auto start server after it is created"` + Image string `help:"New root Image template ID"` + Keypair string `help:"ssh Keypair used for login"` + Password string `help:"Default user password"` } R(&ServerRebuildRootOptions{}, "server-rebuild-root", "Rebuild VM root image with new template", func(s *mcclient.ClientSession, args *ServerRebuildRootOptions) error { params := jsonutils.NewDict() if len(args.Image) > 0 { params.Add(jsonutils.NewString(args.Image), "image_id") } + if args.AutoStart { + params.Add(jsonutils.JSONTrue, "auto_start") + } + if args.Keypair != "" { + params.Add(jsonutils.NewString(args.Keypair), "keypair") + } + if args.Password != "" { + params.Add(jsonutils.NewString(args.Password), "password") + } srv, err := modules.Servers.PerformAction(s, args.SERVER, "rebuild-root", params) if err != nil { return err