From d36a9f8be507945e3994374e518f90c49e53f4e4 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 21 Nov 2018 23:29:35 +0800 Subject: [PATCH 01/10] temp commit --- pkg/cloudprovider/cloudprovider.go | 5 +- pkg/cloudprovider/resources.go | 45 +- pkg/compute/guestdrivers/aliyun.go | 4 +- pkg/compute/guestdrivers/aws.go | 6 +- pkg/compute/guestdrivers/azure.go | 4 +- pkg/compute/guestdrivers/managedvirtual.go | 18 +- pkg/compute/guestdrivers/qcloud.go | 6 +- pkg/compute/hostdrivers/aliyun.go | 16 +- pkg/compute/hostdrivers/aws.go | 14 +- pkg/compute/hostdrivers/azure.go | 14 +- pkg/compute/hostdrivers/kvm.go | 8 +- pkg/compute/hostdrivers/qcloud.go | 14 +- pkg/compute/models/disks.go | 5 +- pkg/compute/models/guests.go | 3 +- pkg/compute/models/hostdrivers.go | 6 +- pkg/compute/models/hosts.go | 235 +++++++-- .../tasks/cloud_provider_sync_info_task.go | 10 + ...cloud_provider_sync_onpremise_info_task.go | 57 +++ pkg/compute/tasks/disk_delete_task.go | 2 +- pkg/compute/tasks/disk_resize_task.go | 2 +- pkg/compute/tasks/guest_create_disk_task.go | 2 +- pkg/compute/tasks/guest_start_task.go | 5 +- pkg/util/aliyun/disk.go | 25 +- pkg/util/aliyun/host.go | 12 + pkg/util/aliyun/instance.go | 19 +- pkg/util/aliyun/provider/provider.go | 8 + pkg/util/aws/disk.go | 13 +- pkg/util/aws/host.go | 12 + pkg/util/aws/instance.go | 27 +- pkg/util/aws/provider/provider.go | 8 + pkg/util/azure/classic_disk.go | 13 +- pkg/util/azure/classic_host.go | 12 + pkg/util/azure/classic_instance.go | 21 +- pkg/util/azure/disk.go | 15 +- pkg/util/azure/host.go | 12 + pkg/util/azure/instance.go | 29 +- pkg/util/azure/provider/provider.go | 8 + pkg/util/esxi/datacenter.go | 26 + pkg/util/esxi/device.go | 92 ++++ pkg/util/esxi/host.go | 221 +++++++-- pkg/util/esxi/hostnic.go | 44 ++ pkg/util/esxi/manager.go | 93 ++-- pkg/util/esxi/mobase.go | 39 +- pkg/util/esxi/provider/provider.go | 25 +- pkg/util/esxi/shell/host.go | 2 +- pkg/util/esxi/shell/store.go | 142 +++++- pkg/util/esxi/shell/virtualmachine.go | 35 ++ pkg/util/esxi/storage.go | 454 +++++++++++++++++- pkg/util/esxi/utils.go | 83 ++++ pkg/util/esxi/vdisk.go | 211 ++++++++ pkg/util/esxi/virtualmachine.go | 263 +++++++++- pkg/util/esxi/vnic.go | 44 ++ pkg/util/printutils/printgetter.go | 12 +- pkg/util/qcloud/disk.go | 17 +- pkg/util/qcloud/host.go | 12 + pkg/util/qcloud/instance.go | 25 +- pkg/util/qcloud/localdisk.go | 13 +- pkg/util/qcloud/provider/provider.go | 8 + pkg/util/vmdkutils/vmdkutils.go | 79 +++ pkg/util/vmdkutils/vmdkutils_test.go | 44 ++ 60 files changed, 2337 insertions(+), 362 deletions(-) create mode 100644 pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go create mode 100644 pkg/util/esxi/device.go create mode 100644 pkg/util/esxi/hostnic.go create mode 100644 pkg/util/esxi/utils.go create mode 100644 pkg/util/esxi/vdisk.go create mode 100644 pkg/util/esxi/vnic.go create mode 100644 pkg/util/vmdkutils/vmdkutils.go create mode 100644 pkg/util/vmdkutils/vmdkutils_test.go diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 22b42736c9..a464146704 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -20,12 +20,15 @@ type ICloudProviderFactory interface { type ICloudProvider interface { GetId() string GetName() string - GetIRegions() []ICloudRegion GetSysInfo() (jsonutils.JSONObject, error) IsPublicCloud() bool + IsOnPremiseInfrastructure() bool + GetIRegions() []ICloudRegion GetIRegionById(id string) (ICloudRegion, error) + GetOnPremiseIHosts() ([]ICloudHost, error) + GetIHostById(id string) (ICloudHost, error) GetIVpcById(id string) (ICloudVpc, error) GetIStorageById(id string) (ICloudStorage, error) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index f0f6b428ab..b988cf9308 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -3,6 +3,7 @@ package cloudprovider import ( "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/util/secrules" @@ -132,10 +133,15 @@ type ICloudHost interface { GetStorageType() string GetHostType() string + GetIsMaintenance() bool + GetVersion() string + GetManagerId() string CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, vswitchId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, extSecGrpId string, userData string) (ICloudVM, error) + + GetIHostNics() ([]ICloudHostNetInterface, error) } type ICloudVM interface { @@ -168,22 +174,22 @@ type ICloudVM interface { // GetSecurityGroup() ICloudSecurityGroup - StartVM() error - StopVM(isForce bool) error - DeleteVM() error + StartVM(ctx context.Context) error + StopVM(ctx context.Context, isForce bool) error + DeleteVM(ctx context.Context) error - UpdateVM(name string) error + UpdateVM(ctx context.Context, name string) error UpdateUserData(userData string) error - RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) + RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) - DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error + DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error - ChangeConfig(instanceId string, ncpu int, vmem int) error + ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) - AttachDisk(diskId string) error - DetachDisk(diskId string) error + AttachDisk(ctx context.Context, diskId string) error + DetachDisk(ctx context.Context, diskId string) error } type ICloudNic interface { @@ -225,7 +231,7 @@ type ICloudDisk interface { ICloudResource IBillingResource - GetIStorge() ICloudStorage + GetIStorage() (ICloudStorage, error) // GetStatus() string GetDiskFormat() string @@ -239,14 +245,14 @@ type ICloudDisk interface { GetDriver() string GetCacheMode() string GetMountpoint() string - Delete() error + Delete(ctx context.Context) error - CreateISnapshot(name string, desc string) (ICloudSnapshot, error) + CreateISnapshot(ctx context.Context, name string, desc string) (ICloudSnapshot, error) GetISnapshot(idStr string) (ICloudSnapshot, error) GetISnapshots() ([]ICloudSnapshot, error) - Resize(newSize int64) error - Reset(snapshotId string) error + Resize(ctx context.Context, newSize int64) error + Reset(ctx context.Context, snapshotId string) error } type ICloudSnapshot interface { @@ -306,3 +312,14 @@ type ICloudNetwork interface { GetAllocTimeoutSeconds() int } + +type ICloudHostNetInterface interface { + GetDevice() string + GetDriver() string + GetMac() string + GetIndex() int8 + IsLinkUp() bool + GetIpAddr() string + GetMtu() int16 + GetNicType() string +} diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index ae607e644e..9f119b98b2 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -339,7 +339,7 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu } } - err := iVM.DeployVM(name, passwd, publicKey, deleteKeypair, description) + err := iVM.DeployVM(ctx, name, passwd, publicKey, deleteKeypair, description) if err != nil { return nil, err } @@ -378,7 +378,7 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu } } - diskId, err := iVM.RebuildRoot(desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) + diskId, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) if err != nil { return nil, err } diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index 76449f0b64..8524bfe0c7 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -153,7 +153,7 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - err := iVM.DeployVM(name, passwd, publicKey, deleteKeypair, description) + err := iVM.DeployVM(ctx, name, passwd, publicKey, deleteKeypair, description) if err != nil { return nil, err } @@ -169,7 +169,7 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest } taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - diskId, err := iVM.RebuildRoot(desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) + diskId, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) if err != nil { return nil, err } @@ -298,7 +298,7 @@ func (self *SAwsGuestDriver) RequestDiskSnapshot(ctx context.Context, guest *mod iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) snapshot := iSnapshot.(*models.SSnapshot) taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - cloudSnapshot, err := providerDisk.CreateISnapshot(snapshot.Name, "") + cloudSnapshot, err := providerDisk.CreateISnapshot(ctx, snapshot.Name, "") if err != nil { return nil, err } diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index 139dbad0e5..3e39359acc 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -188,7 +188,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue deleteKeypair := jsonutils.QueryBoolean(params, "__delete_keypair__", false) taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - err := iVM.DeployVM(name, passwd, publicKey, deleteKeypair, description) + err := iVM.DeployVM(ctx, name, passwd, publicKey, deleteKeypair, description) if err != nil { return nil, err } @@ -203,7 +203,7 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue } taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - _, err := iVM.RebuildRoot(desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) + _, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) if err != nil { return nil, err } diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index f3e7458173..aaaf6e165b 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -120,7 +120,7 @@ func (self *SManagedVirtualizedGuestDriver) OnGuestDeployTaskDataReceived(ctx co return nil } -func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(_ context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { +func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { ihost, e := host.GetIHost() if e != nil { return nil, e @@ -133,7 +133,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(_ context.Context result := jsonutils.NewDict() if ivm.GetStatus() != models.VM_RUNNING { - if err := ivm.StartVM(); err != nil { + if err := ivm.StartVM(ctx); err != nil { return nil, e } else { task.ScheduleRun(result) @@ -159,7 +159,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte return nil, err } } - err = ivm.DeleteVM() + err = ivm.DeleteVM(ctx) if err != nil { return nil, err } @@ -174,7 +174,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte return nil, err } } - err = idisk.Delete() + err = idisk.Delete(ctx) if err != nil { return nil, err } @@ -195,7 +195,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestStopOnHost(ctx context.Contex if err != nil { return nil, err } - err = ivm.StopVM(true) + err = ivm.StopVM(ctx, true) return nil, err }) return nil @@ -297,7 +297,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestChangeVmConfig(ctx context.Co } if int(guest.VcpuCount) != config.Cpu || guest.VmemSize != config.Memory { - err = iVM.ChangeConfig(config.InstanceId, config.Cpu, config.Memory) + err = iVM.ChangeConfig(ctx, config.InstanceId, config.Cpu, config.Memory) if err != nil { return err } @@ -335,12 +335,12 @@ func (self *SManagedVirtualizedGuestDriver) RequestSyncConfigOnHost(ctx context. return nil, err } for _, disk := range removed { - if err := iVM.DetachDisk(disk.GetId()); err != nil { + if err := iVM.DetachDisk(ctx, disk.GetId()); err != nil { return nil, err } } for _, disk := range added { - if err := iVM.AttachDisk(disk.ExternalId); err != nil { + if err := iVM.AttachDisk(ctx, disk.ExternalId); err != nil { return nil, err } } @@ -362,7 +362,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestDiskSnapshot(ctx context.Cont iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) snapshot := iSnapshot.(*models.SSnapshot) taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - cloudSnapshot, err := providerDisk.CreateISnapshot(snapshot.Name, "") + cloudSnapshot, err := providerDisk.CreateISnapshot(ctx, snapshot.Name, "") if err != nil { return nil, err } diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index 9c523c085c..a1f34041af 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -196,7 +196,7 @@ func (self *SQcloudGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu // } // } - err := iVM.DeployVM(name, passwd, publicKey, deleteKeypair, description) + err := iVM.DeployVM(ctx, name, passwd, publicKey, deleteKeypair, description) if err != nil { return nil, err } @@ -221,7 +221,7 @@ func (self *SQcloudGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu // } // } - diskId, err := iVM.RebuildRoot(desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) + diskId, err := iVM.RebuildRoot(ctx, desc.ExternalImageId, passwd, publicKey, desc.SysDiskSize) if err != nil { return nil, err } @@ -356,7 +356,7 @@ func (self *SQcloudGuestDriver) RequestDiskSnapshot(ctx context.Context, guest * iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) snapshot := iSnapshot.(*models.SSnapshot) taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - cloudSnapshot, err := providerDisk.CreateISnapshot(snapshot.Name, "") + cloudSnapshot, err := providerDisk.CreateISnapshot(ctx, snapshot.Name, "") if err != nil { return nil, err } diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index 2c9df47e38..bff39e84af 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -82,7 +82,7 @@ func (self *SAliyunHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) } else { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - if snapshot, err := iDisk.CreateISnapshot(fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { + if snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { return nil, err } else { params := task.GetParams() @@ -165,7 +165,7 @@ func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, return nil } -func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { +func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { data := jsonutils.NewDict() if iCloudStorage, err := storage.GetIStorage(); err != nil { return err @@ -175,23 +175,23 @@ func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, s return nil } return err - } else if err := iDisk.Delete(); err != nil { + } else if err := iDisk.Delete(ctx); err != nil { return err } task.ScheduleRun(data) return nil } -func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(host, storage, disk, size, task) +func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) } -func (self *SAliyunHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { +func (self *SAliyunHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { if iCloudStorage, err := storage.GetIStorage(); err != nil { return err } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { return err - } else if err := iDisk.Resize(size >> 10); err != nil { + } else if err := iDisk.Resize(ctx, size>>10); err != nil { return err } else { task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) @@ -209,7 +209,7 @@ func (self *SAliyunHostDriver) RequestResetDisk(ctx context.Context, host *model return err } taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - err := iDisk.Reset(snapshotId) + err := iDisk.Reset(ctx, snapshotId) return nil, err }) return nil diff --git a/pkg/compute/hostdrivers/aws.go b/pkg/compute/hostdrivers/aws.go index b4022ab211..5488a5667d 100644 --- a/pkg/compute/hostdrivers/aws.go +++ b/pkg/compute/hostdrivers/aws.go @@ -76,7 +76,7 @@ func (self *SAwsHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, ho return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) } else { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - if snapshot, err := iDisk.CreateISnapshot(fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { + if snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { return nil, err } else { params := task.GetParams() @@ -154,7 +154,7 @@ func (self *SAwsHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, ho return nil } -func (self *SAwsHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { +func (self *SAwsHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { data := jsonutils.NewDict() if iCloudStorage, err := storage.GetIStorage(); err != nil { return err @@ -164,23 +164,23 @@ func (self *SAwsHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, stor return nil } return err - } else if err := iDisk.Delete(); err != nil { + } else if err := iDisk.Delete(ctx); err != nil { return err } task.ScheduleRun(data) return nil } -func (self *SAwsHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(host, storage, disk, size, task) +func (self *SAwsHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) } -func (self *SAwsHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { +func (self *SAwsHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { if iCloudStorage, err := storage.GetIStorage(); err != nil { return err } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { return err - } else if err := iDisk.Resize(size >> 10); err != nil { + } else if err := iDisk.Resize(ctx, size>>10); err != nil { return err } else { task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) diff --git a/pkg/compute/hostdrivers/azure.go b/pkg/compute/hostdrivers/azure.go index 6635e9c879..93581da592 100644 --- a/pkg/compute/hostdrivers/azure.go +++ b/pkg/compute/hostdrivers/azure.go @@ -115,7 +115,7 @@ func (self *SAzureHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, return nil } -func (self *SAzureHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { +func (self *SAzureHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { data := jsonutils.NewDict() if iCloudStorage, err := storage.GetIStorage(); err != nil { return err @@ -125,23 +125,23 @@ func (self *SAzureHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, st return nil } return err - } else if err := iDisk.Delete(); err != nil { + } else if err := iDisk.Delete(ctx); err != nil { return err } task.ScheduleRun(data) return nil } -func (self *SAzureHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(host, storage, disk, size, task) +func (self *SAzureHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) } -func (self *SAzureHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { +func (self *SAzureHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { if iCloudStorage, err := storage.GetIStorage(); err != nil { return err } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { return err - } else if err := iDisk.Resize(size >> 10); err != nil { + } else if err := iDisk.Resize(ctx, size>>10); err != nil { return err } else { task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) @@ -163,7 +163,7 @@ func (self *SAzureHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) } else { taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - if snapshot, err := iDisk.CreateISnapshot(fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { + if snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { return nil, err } else { params := task.GetParams() diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 57e9fad69a..93b5cab4b6 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -90,7 +90,7 @@ func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, ho return err } -func (self *SKVMHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { +func (self *SKVMHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { log.Infof("Deallocating disk on host %s", host.GetName()) header := http.Header{} header.Add("X-Task-Id", task.GetTaskId()) @@ -101,7 +101,7 @@ func (self *SKVMHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, stor return err } -func (self *SKVMHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { +func (self *SKVMHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { header := http.Header{} header.Add("X-Task-Id", task.GetTaskId()) header.Add("X-Region-Version", "v2") @@ -114,8 +114,8 @@ func (self *SKVMHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage return err } -func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - self.RequestResizeDiskOnHost(host, storage, disk, size, task) +func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) header := http.Header{} header.Add("X-Task-Id", task.GetTaskId()) header.Add("X-Region-Version", "v2") diff --git a/pkg/compute/hostdrivers/qcloud.go b/pkg/compute/hostdrivers/qcloud.go index 536c620c64..7a30ca306a 100644 --- a/pkg/compute/hostdrivers/qcloud.go +++ b/pkg/compute/hostdrivers/qcloud.go @@ -107,7 +107,7 @@ func (self *SQcloudHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, return nil } -func (self *SQcloudHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { +func (self *SQcloudHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { data := jsonutils.NewDict() if iCloudStorage, err := storage.GetIStorage(); err != nil { return err @@ -117,18 +117,18 @@ func (self *SQcloudHostDriver) RequestDeallocateDiskOnHost(host *models.SHost, s return nil } return err - } else if err := iDisk.Delete(); err != nil { + } else if err := iDisk.Delete(ctx); err != nil { return err } task.ScheduleRun(data) return nil } -func (self *SQcloudHostDriver) RequestResizeDiskOnHostOnline(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(host, storage, disk, size, task) +func (self *SQcloudHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) } -func (self *SQcloudHostDriver) RequestResizeDiskOnHost(host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { +func (self *SQcloudHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { iCloudStorage, err := storage.GetIStorage() if err != nil { return err @@ -137,7 +137,7 @@ func (self *SQcloudHostDriver) RequestResizeDiskOnHost(host *models.SHost, stora if err != nil { return err } - err = iDisk.Resize(size >> 10) + err = iDisk.Resize(ctx, size>>10) if err != nil { return err } @@ -164,7 +164,7 @@ func (self *SQcloudHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) } taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - snapshot, err := iDisk.CreateISnapshot(fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage") + snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage") if err != nil { return nil, err } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 13fad325ae..c5bf013aca 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -754,7 +754,8 @@ func (manager *SDiskManager) syncCloudDisk(ctx context.Context, userCred mcclien diskObj, err := manager.FetchByExternalId(vdisk.GetGlobalId()) if err != nil { if err == sql.ErrNoRows { - vstorage := vdisk.GetIStorge() + vstorage, _ := vdisk.GetIStorage() + storageObj, err := StorageManager.FetchByExternalId(vstorage.GetGlobalId()) if err != nil { log.Errorf("cannot find storage of vdisk %s", err) @@ -1101,6 +1102,7 @@ type DiskInfo struct { MediumType string Driver string Cache string + DiskType string } func (self *SDisk) ToDiskInfo() DiskInfo { @@ -1110,6 +1112,7 @@ func (self *SDisk) ToDiskInfo() DiskInfo { MountPoint: self.GetMountPoint(), Format: self.DiskFormat, Size: int64(self.DiskSize), + DiskType: self.DiskType, } storage := self.GetStorage() if storage == nil { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 208e5a5ce3..fef2cebfc6 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3915,6 +3915,7 @@ func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { diskSpec.Set("size", jsonutils.NewInt(info.Size)) diskSpec.Set("backend", jsonutils.NewString(info.Backend)) diskSpec.Set("medium_type", jsonutils.NewString(info.MediumType)) + diskSpec.Set("disk_type", jsonutils.NewString(info.DiskType)) diskSpecs.Add(diskSpec) } spec.Set("disk", diskSpecs) @@ -4040,7 +4041,7 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { templateId := self.GetTemplateId() if len(templateId) > 0 { - desc.Set("cpu", jsonutils.NewString(templateId)) + desc.Set("template_id", jsonutils.NewString(templateId)) } extBw := self.getBandwidth(true) intBw := self.getBandwidth(false) diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go index d6217e3c69..606f730639 100644 --- a/pkg/compute/models/hostdrivers.go +++ b/pkg/compute/models/hostdrivers.go @@ -17,9 +17,9 @@ type IHostDriver interface { RequestPrepareSaveDiskOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask) error RequestSaveUploadImageOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error RequestAllocateDiskOnStorage(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask, content *jsonutils.JSONDict) error - RequestDeallocateDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error - RequestResizeDiskOnHostOnline(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error - RequestResizeDiskOnHost(host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error + RequestDeallocateDiskOnHost(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error + RequestResizeDiskOnHostOnline(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error + RequestResizeDiskOnHost(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error RequestDeleteSnapshotsWithStorage(ctx context.Context, host *SHost, snapshot *SSnapshot, task taskman.ITask) error RequestResetDisk(ctx context.Context, host *SHost, disk *SDisk, params *jsonutils.JSONDict, task taskman.ITask) error RequestCleanUpDiskSnapshots(ctx context.Context, host *SHost, disk *SDisk, params *jsonutils.JSONDict, task taskman.ITask) error diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 5a52ab1505..2f57efb7e0 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "database/sql" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -944,9 +945,12 @@ func (self *SHost) GetHostDriver() IHostDriver { return GetHostDriver(self.HostType) } -func (manager *SHostManager) getHostsByZone(zone *SZone, provider *SCloudprovider) ([]SHost, error) { +func (manager *SHostManager) getHostsByZoneProvider(zone *SZone, provider *SCloudprovider) ([]SHost, error) { hosts := make([]SHost, 0) - q := manager.Query().Equals("zone_id", zone.Id) + q := manager.Query() + if zone != nil { + q = q.Equals("zone_id", zone.Id) + } if provider != nil { q = q.Equals("manager_id", provider.Id) } @@ -963,7 +967,7 @@ func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.To remoteHosts := make([]cloudprovider.ICloudHost, 0) syncResult := compare.SyncResult{} - dbHosts, err := manager.getHostsByZone(zone, provider) + dbHosts, err := manager.getHostsByZoneProvider(zone, provider) if err != nil { syncResult.Error(err) return nil, nil, syncResult @@ -1028,6 +1032,7 @@ func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.To func (self *SHost) syncWithCloudHost(extHost cloudprovider.ICloudHost) error { _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.Name = extHost.GetName() + self.Status = extHost.GetStatus() self.HostStatus = extHost.GetHostStatus() self.AccessIp = extHost.GetAccessIp() @@ -1047,6 +1052,9 @@ func (self *SHost) syncWithCloudHost(extHost cloudprovider.ICloudHost) error { self.IsEmulated = extHost.IsEmulated() self.Enabled = extHost.GetEnabled() + self.IsMaintenance = extHost.GetIsMaintenance() + self.Version = extHost.GetVersion() + return nil }) if err != nil { @@ -1055,13 +1063,23 @@ func (self *SHost) syncWithCloudHost(extHost cloudprovider.ICloudHost) error { return err } -func (manager *SHostManager) newFromCloudHost(extHost cloudprovider.ICloudHost, zone *SZone) (*SHost, error) { +func (manager *SHostManager) newFromCloudHost(extHost cloudprovider.ICloudHost, izone *SZone) (*SHost, error) { host := SHost{} host.SetModelManager(manager) + if izone == nil { + wire, err := WireManager.GetWireOfIp(extHost.GetAccessIp()) + if err != nil { + msg := fmt.Sprintf("fail to find wire for host %s %s: %s", extHost.GetName(), extHost.GetAccessIp(), err) + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + izone = wire.GetZone() + } + host.Name = extHost.GetName() host.ExternalId = extHost.GetGlobalId() - host.ZoneId = zone.Id + host.ZoneId = izone.Id // host.ManagerId = extHost.GetManagerId() host.HostType = extHost.GetHostType() @@ -1084,6 +1102,9 @@ func (manager *SHostManager) newFromCloudHost(extHost cloudprovider.ICloudHost, host.ManagerId = extHost.GetManagerId() host.IsEmulated = extHost.IsEmulated() + host.IsMaintenance = extHost.GetIsMaintenance() + host.Version = extHost.GetVersion() + err := manager.TableSpec().Insert(&host) if err != nil { log.Errorf("newFromCloudHost fail %s", err) @@ -1177,8 +1198,18 @@ func (self *SHost) Attach2Storage(ctx context.Context, userCred mcclient.TokenCr func (self *SHost) newCloudHostStorage(ctx context.Context, userCred mcclient.TokenCredential, extStorage cloudprovider.ICloudStorage) error { storageObj, err := StorageManager.FetchByExternalId(extStorage.GetGlobalId()) if err != nil { - log.Errorf("%s", err) - return nil + if err == sql.ErrNoRows { + // no cloud storage found, this may happen for on-premise host + // create the storage + storageObj, err = StorageManager.newFromCloudStorage(extStorage, self.GetZone()) + if err != nil { + log.Errorf("create by cloud storage fail %s", err) + return err + } + } else { + log.Errorf("%s", err) + return err + } } storage := storageObj.(*SStorage) err = self.Attach2Storage(ctx, userCred, storage, "") @@ -2377,26 +2408,49 @@ func (self *SHost) PerformAddNetif(ctx context.Context, userCred mcclient.TokenC } wire, _ := data.GetString("wire") ipAddr, _ := data.GetString("ip_addr") + rate, _ := data.Int("rate") + nicType, _ := data.GetString("nic_type") + index, _ := data.Int("index") + linkUp, _ := data.GetString("link_up") + mtu, _ := data.Int("mtu") + reset := jsonutils.QueryBoolean(data, "reset", false) + strInterface, _ := data.GetString("interface") + bridge, _ := data.GetString("bridge") + reserve := jsonutils.QueryBoolean(data, "reserve", false) + requireDesignatedIp := jsonutils.QueryBoolean(data, "require_designated_ip", false) + + err := self.addNetif(ctx, userCred, mac, wire, ipAddr, int(rate), nicType, int8(index), utils.ToBool(linkUp), + int16(mtu), reset, strInterface, bridge, reserve, requireDesignatedIp) + return nil, err +} + +func (self *SHost) addNetif(ctx context.Context, userCred mcclient.TokenCredential, + mac string, wire string, ipAddr string, + rate int, nicType string, index int8, linkUp bool, mtu int16, + reset bool, strInterface string, bridge string, + reserve bool, requireDesignatedIp bool, +) error { + var sw *SWire if len(wire) > 0 && len(ipAddr) == 0 { iWire, err := WireManager.FetchByIdOrName(userCred, wire) if err != nil { - return nil, httperrors.NewBadRequestError("Wire %s not found", wire) + return httperrors.NewBadRequestError("Wire %s not found", wire) } sw = iWire.(*SWire) } else if len(ipAddr) > 0 && len(wire) == 0 { ipWire, err := WireManager.GetWireOfIp(ipAddr) if err != nil { - return nil, httperrors.NewBadRequestError("IP %s not attach to any wire", ipAddr) + return httperrors.NewBadRequestError("IP %s not attach to any wire", ipAddr) } sw = ipWire } else if len(wire) > 0 && len(ipAddr) > 0 { ipWire, err := WireManager.GetWireOfIp(ipAddr) if err != nil { - return nil, httperrors.NewBadRequestError("IP %s not attach to any wire", ipAddr) + return httperrors.NewBadRequestError("IP %s not attach to any wire", ipAddr) } if ipWire.Id != wire && ipWire.GetName() != wire { - httperrors.NewBadRequestError("IP %s not attach to wire %s", ipAddr, wire) + return httperrors.NewBadRequestError("IP %s not attach to wire %s", ipAddr, wire) } sw = ipWire } @@ -2408,24 +2462,14 @@ func (self *SHost) PerformAddNetif(ctx context.Context, userCred mcclient.TokenC if sw != nil { netif.WireId = sw.Id } - if rate, err := data.Int("rate"); err == nil { - netif.Rate = int(rate) - } - if nicType, err := data.GetString("nic_type"); err == nil { - netif.NicType = nicType - } - if index, err := data.Int("index"); err == nil { - netif.Index = int8(index) - } - if linkUp, err := data.GetString("link_up"); err == nil { - netif.LinkUp = utils.ToBool(linkUp) - } - if mtu, err := data.Int("mtu"); err == nil { - netif.Mtu = int16(mtu) - } + netif.Rate = rate + netif.NicType = nicType + netif.Index = index + netif.LinkUp = linkUp + netif.Mtu = mtu err = NetInterfaceManager.TableSpec().Insert(netif) if err != nil { - return nil, err + return err } } else { var changed = false @@ -2438,44 +2482,42 @@ func (self *SHost) PerformAddNetif(ctx context.Context, userCred mcclient.TokenC changed = true netif.WireId = sw.Id } - if rate, err := data.Int("rate"); err == nil { + if rate != netif.Rate { changed = true netif.Rate = int(rate) } - if nicType, err := data.GetString("nic_type"); err == nil { + if nicType != netif.NicType { changed = true netif.NicType = nicType } - if index, err := data.Int("index"); err == nil && index >= 0 { + if index >= 0 && index != netif.Index { changed = true netif.Index = int8(index) } - if linkUp, err := data.GetString("link_up"); err != nil { + if linkUp != netif.LinkUp { changed = true - netif.LinkUp = utils.ToBool(linkUp) + netif.LinkUp = linkUp } - if mtu, err := data.Int("mtu"); err != nil { + if mtu != netif.Mtu { changed = true netif.Mtu = int16(mtu) } return nil }) if err != nil { - return nil, err + return err } - if changed || jsonutils.QueryBoolean(data, "reset", false) { + if changed || reset { self.DisableNetif(ctx, userCred, netif, false) } } sw = netif.GetWire() if sw != nil { - strInterface, err := data.GetString("interface") - if err != nil { + if len(strInterface) == 0 { strInterface = fmt.Sprintf("eth%d", netif.Index) } if len(strInterface) > 0 { - bridge, err := data.GetString("bridge") - if err != nil { + if len(bridge) == 0 { bridge = fmt.Sprintf("br%s", sw.GetName()) } var isMaster = netif.NicType == NIC_TYPE_ADMIN @@ -2490,7 +2532,7 @@ func (self *SHost) PerformAddNetif(ctx context.Context, userCred mcclient.TokenC hw.MacAddr = mac err := HostwireManager.TableSpec().Insert(hw) if err != nil { - return nil, err + return err } } else { hw := ihw.(*SHostwire) @@ -2504,15 +2546,13 @@ func (self *SHost) PerformAddNetif(ctx context.Context, userCred mcclient.TokenC } } } - reserve := jsonutils.QueryBoolean(data, "reserve", false) - requireDesignatedIp := jsonutils.QueryBoolean(data, "require_designated_ip", false) if len(ipAddr) > 0 { err = self.EnableNetif(ctx, userCred, netif, "", ipAddr, "", reserve, requireDesignatedIp) if err != nil { - return nil, httperrors.NewBadRequestError(err.Error()) + return httperrors.NewBadRequestError(err.Error()) } } - return nil, nil + return nil } func (self *SHost) AllowPerformEnableNetif(ctx context.Context, @@ -2963,3 +3003,110 @@ func (self *SHost) UpdateDiskConfig(layouts []baremetal.Layout) error { } return nil } + +func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.TokenCredential, ihost cloudprovider.ICloudHost) compare.SyncResult { + result := compare.SyncResult{} + + netIfs := host.GetNetInterfaces() + extNics, err := ihost.GetIHostNics() + if err != nil { + result.Error(err) + return result + } + + disables := make([]*SNetInterface, 0) + enables := make([]cloudprovider.ICloudHostNetInterface, 0) + + type sRemoveNetInterface struct { + netif *SNetInterface + reserveIp bool + } + + type sAddNetInterface struct { + netif cloudprovider.ICloudHostNetInterface + reserveIp bool + } + + removes := make([]sRemoveNetInterface, 0) + adds := make([]sAddNetInterface, 0) + + nicMax := len(netIfs) + if nicMax < len(extNics) { + nicMax = len(extNics) + } + for i := 0; i < nicMax; i += 1 { + if i < len(netIfs) && i < len(extNics) { + obn := netIfs[i].GetBaremetalNetwork() + var oip string + if obn != nil { + oip = obn.IpAddr + } + nip := extNics[i].GetIpAddr() + if netIfs[i].Mac == extNics[i].GetMac() { + if oip != nip { + if obn != nil { + disables = append(disables, &netIfs[i]) + } + if len(nip) > 0 { + enables = append(enables, extNics[i]) + } + } else { + // do nothing, in sync + } + } else { + reserveIp := false + if len(oip) > 0 && oip == nip { + // # mac change case + reserveIp = true + } + removes = append(removes, sRemoveNetInterface{netif: &netIfs[i], reserveIp: reserveIp}) + adds = append(adds, sAddNetInterface{netif: extNics[i], reserveIp: reserveIp}) + } + } else if i < len(netIfs) && i >= len(extNics) { + removes = append(removes, sRemoveNetInterface{netif: &netIfs[i], reserveIp: false}) + } else if i >= len(netIfs) && i < len(extNics) { + adds = append(adds, sAddNetInterface{netif: extNics[i], reserveIp: false}) + } + } + + for i := len(removes) - 1; i >= 0; i -= 1 { + err = host.RemoveNetif(ctx, userCred, removes[i].netif, removes[i].reserveIp) + if err != nil { + result.DeleteError(err) + } else { + result.Delete() + } + } + + for i := len(disables) - 1; i >= 0; i -= 1 { + err = host.DisableNetif(ctx, userCred, disables[i], false) + if err != nil { + result.DeleteError(err) + } else { + result.Delete() + } + } + + for i := 0; i < len(enables); i += 1 { + netif := host.GetNetInterface(enables[i].GetMac()) + err = host.EnableNetif(ctx, userCred, netif, "", enables[i].GetIpAddr(), "", false, true) + if err != nil { + result.AddError(err) + } else { + result.Add() + } + } + + for i := 0; i < len(adds); i += 1 { + extNic := adds[i].netif + err = host.addNetif(ctx, userCred, extNic.GetMac(), "", extNic.GetIpAddr(), 0, extNic.GetNicType(), extNic.GetIndex(), + extNic.IsLinkUp(), extNic.GetMtu(), false, "", "", false, true) + if err != nil { + result.AddError(err) + } else { + result.Add() + } + } + + return result +} diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index 18e75d4d03..db48b60a96 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -93,7 +93,17 @@ func syncCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider, notes := fmt.Sprintf("Start sync host info ...") log.Infof(notes) db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_START, "", task.UserCred) + + if driver.IsOnPremiseInfrastructure() { + syncOnPremiseCloudProviderInfo(ctx, provider, task, driver, syncRange) + } else { + syncPublicCloudProviderInfo(ctx, provider, task, driver, syncRange) + } +} + +func syncPublicCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, driver cloudprovider.ICloudProvider, syncRange *models.SSyncRange) { regions := driver.GetIRegions() + localRegions, remoteRegions, result := models.CloudregionManager.SyncRegions(ctx, task.UserCred, provider.Provider, regions) msg := result.Result() log.Infof("SyncRegion result: %s", msg) diff --git a/pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go b/pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go new file mode 100644 index 0000000000..b0f07904ed --- /dev/null +++ b/pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go @@ -0,0 +1,57 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +func syncOnPremiseCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, driver cloudprovider.ICloudProvider, syncRange *models.SSyncRange) { + ihosts, err := driver.GetOnPremiseIHosts() + if err != nil { + msg := fmt.Sprintf("GetOnPremiseIHosts for provider %s failed %s", provider.GetName(), err) + log.Errorf(msg) + logSyncFailed(provider, task, msg) + return + } + + localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, nil, ihosts) + msg := result.Result() + notes := fmt.Sprintf("SyncHosts for provider %s result: %s", provider.Name, msg) + log.Infof(notes) + if result.IsError() { + logSyncFailed(provider, task, msg) + return + } + db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred) + logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) + + for i := 0; i < len(localHosts); i += 1 { + if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) { + continue + } + syncHostStorages(ctx, provider, task, &localHosts[i], remoteHosts[i]) + syncHostNics(ctx, provider, task, &localHosts[i], remoteHosts[i]) + syncHostVMs(ctx, provider, task, &localHosts[i], remoteHosts[i], syncRange) + } +} + +func syncHostNics(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, localHost *models.SHost, remoteHost cloudprovider.ICloudHost) { + result := localHost.SyncHostExternalNics(ctx, task.GetUserCred(), remoteHost) + msg := result.Result() + notes := fmt.Sprintf("SyncHostWires for host %s result: %s", localHost.Name, msg) + log.Infof(notes) + if result.IsError() { + logSyncFailed(provider, task, msg) + return + } + db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.GetUserCred()) + logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) +} diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index ff6c4f76c1..9c4efd11c8 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -54,7 +54,7 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD self.SetStage("on_guest_disk_delete_succ", nil) if host == nil { self.OnGuestDiskDeleteFailed(ctx, disk, httperrors.NewNotFoundError("fail to find master host")) - } else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(host, storage, disk, self); err != nil { + } else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self); err != nil { self.OnGuestDiskDeleteFailed(ctx, disk, err) } } diff --git a/pkg/compute/tasks/disk_resize_task.go b/pkg/compute/tasks/disk_resize_task.go index 3fff258569..9724a171d8 100644 --- a/pkg/compute/tasks/disk_resize_task.go +++ b/pkg/compute/tasks/disk_resize_task.go @@ -54,7 +54,7 @@ func (self *DiskResizeTask) StartResizeDisk(ctx context.Context, host *models.SH if online { proc = host.GetHostDriver().RequestResizeDiskOnHostOnline } - if err := proc(host, storage, disk, size, self); err != nil { + if err := proc(ctx, host, storage, disk, size, self); err != nil { log.Errorf("request_resize_disk_on_host: %v", err) self.OnStartResizeDiskFailed(ctx, disk, err) return diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index 7514bb17de..76502bc2e3 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -196,7 +196,7 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex return } - err = iVM.AttachDisk(disk.GetExternalId()) + err = iVM.AttachDisk(ctx, disk.GetExternalId()) if err != nil { log.Debugf("Attach Disk %s to guest fail: %s", diskId, err) self.SetStageFailed(ctx, "Attach Disk to guest fail") diff --git a/pkg/compute/tasks/guest_start_task.go b/pkg/compute/tasks/guest_start_task.go index 99dae67cf3..b8ff8f16ff 100644 --- a/pkg/compute/tasks/guest_start_task.go +++ b/pkg/compute/tasks/guest_start_task.go @@ -48,8 +48,9 @@ func (self *GuestStartTask) startStart(ctx context.Context, guest *models.SGuest self.onStartGuestFailed(ctx, guest, err) } else { if result != nil && jsonutils.QueryBoolean(result, "is_running", false) { - guest.SetStatus(self.UserCred, models.VM_RUNNING, "start") - self.taskComplete(ctx, guest) + // guest.SetStatus(self.UserCred, models.VM_RUNNING, "start") + // self.taskComplete(ctx, guest) + self.OnStartComplete(ctx, guest, nil) } } } diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index 4b31697c2e..c1189f130c 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -108,16 +109,20 @@ func (self *SDisk) GetId() string { return self.DiskId } -func (self *SDisk) Delete() error { - if _, err := self.storage.zone.region.getDisk(self.DiskId); err == cloudprovider.ErrNotFound { - // 未找到disk, 说明disk已经被删除了. 避免回收站中disk-delete循环删除失败 - log.Errorf("Failed to find disk %s when delete", self.DiskId) - return nil +func (self *SDisk) Delete(ctx context.Context) error { + _, err := self.storage.zone.region.getDisk(self.DiskId) + if err != nil { + if err == cloudprovider.ErrNotFound { + // 未找到disk, 说明disk已经被删除了. 避免回收站中disk-delete循环删除失败 + return nil + } + log.Errorf("Failed to find disk %s when delete: %s", self.DiskId, err) + return err } return self.storage.zone.region.deleteDisk(self.DiskId) } -func (self *SDisk) Resize(size int64) error { +func (self *SDisk) Resize(ctx context.Context, size int64) error { return self.storage.zone.region.resizeDisk(self.DiskId, size) } @@ -136,8 +141,8 @@ func (self *SDisk) IsEmulated() bool { return false } -func (self *SDisk) GetIStorge() cloudprovider.ICloudStorage { - return self.storage +func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return self.storage, nil } func (self *SDisk) GetStatus() string { @@ -284,7 +289,7 @@ func (self *SRegion) resetDisk(diskId, snapshotId string) error { return nil } -func (self *SDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) { +func (self *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { if snapshotId, err := self.storage.zone.region.CreateSnapshot(self.DiskId, name, desc); err != nil { log.Errorf("createSnapshot fail %s", err) return nil, err @@ -354,7 +359,7 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return isnapshots, nil } -func (self *SDisk) Reset(snapshotId string) error { +func (self *SDisk) Reset(ctx context.Context, snapshotId string) error { return self.storage.zone.region.resetDisk(self.DiskId, snapshotId) } diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index ad1e28e7a4..bd4d3a6a08 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -272,3 +272,15 @@ func (self *SHost) _createVM(name string, imgId string, sysDiskSize int, cpu int return "", fmt.Errorf("Failed to create, specification not supported") } + +func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (host *SHost) GetIsMaintenance() bool { + return false +} + +func (host *SHost) GetVersion() string { + return ALIYUN_API_VERSION +} diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index f43fe0e133..52aa481a3e 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" + "context" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -382,7 +383,7 @@ func (self *SInstance) GetHypervisor() string { return models.HYPERVISOR_ALIYUN } -func (self *SInstance) StartVM() error { +func (self *SInstance) StartVM(ctx context.Context) error { timeout := 300 * time.Second interval := 15 * time.Second @@ -406,7 +407,7 @@ func (self *SInstance) StartVM() error { return cloudprovider.ErrTimeout } -func (self *SInstance) StopVM(isForce bool) error { +func (self *SInstance) StopVM(ctx context.Context, isForce bool) error { err := self.host.zone.region.StopVM(self.InstanceId, isForce) if err != nil { return err @@ -432,11 +433,11 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { return ret, nil } -func (self *SInstance) UpdateVM(name string) error { +func (self *SInstance) UpdateVM(ctx context.Context, name string) error { return self.host.zone.region.UpdateVM(self.InstanceId, name) } -func (self *SInstance) DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error { +func (self *SInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { var keypairName string if len(publicKey) > 0 { var err error @@ -449,7 +450,7 @@ func (self *SInstance) DeployVM(name string, password string, publicKey string, return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, deleteKeypair, description) } -func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { keypair := "" if len(publicKey) > 0 { var err error @@ -466,15 +467,15 @@ func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey stri return diskId, nil } -func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil) } -func (self *SInstance) AttachDisk(diskId string) error { +func (self *SInstance) AttachDisk(ctx context.Context, diskId string) error { return self.host.zone.region.AttachDisk(self.InstanceId, diskId) } -func (self *SInstance) DetachDisk(diskId string) error { +func (self *SInstance) DetachDisk(ctx context.Context, diskId string) error { return self.host.zone.region.DetachDisk(self.InstanceId, diskId) } @@ -693,7 +694,7 @@ func (self *SRegion) DeployVM(instanceId string, name string, password string, k } } -func (self *SInstance) DeleteVM() error { +func (self *SInstance) DeleteVM(ctx context.Context) error { for { err := self.host.zone.region.DeleteVM(self.InstanceId) if err != nil { diff --git a/pkg/util/aliyun/provider/provider.go b/pkg/util/aliyun/provider/provider.go index 5304c8aed5..fe5a4a1813 100644 --- a/pkg/util/aliyun/provider/provider.go +++ b/pkg/util/aliyun/provider/provider.go @@ -55,6 +55,10 @@ func (self *SAliyunProvider) IsPublicCloud() bool { return true } +func (self *SAliyunProvider) IsOnPremiseInfrastructure() bool { + return false +} + func (self *SAliyunProvider) GetId() string { return aliyun.CLOUD_PROVIDER_ALIYUN } @@ -106,3 +110,7 @@ func (self *SAliyunProvider) GetBalance() (float64, error) { } return balance.AvailableAmount, nil } + +func (self *SAliyunProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/aws/disk.go b/pkg/util/aws/disk.go index edef37c559..8cf0d4f1dc 100644 --- a/pkg/util/aws/disk.go +++ b/pkg/util/aws/disk.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "context" "github.com/aws/aws-sdk-go/service/ec2" "github.com/coredns/coredns/plugin/pkg/log" "yunion.io/x/jsonutils" @@ -117,8 +118,8 @@ func (self *SDisk) GetExpiredAt() time.Time { return self.ExpiredTime } -func (self *SDisk) GetIStorge() cloudprovider.ICloudStorage { - return self.storage +func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return self.storage, nil } func (self *SDisk) GetDiskFormat() string { @@ -161,7 +162,7 @@ func (self *SDisk) GetMountpoint() string { return "" } -func (self *SDisk) Delete() error { +func (self *SDisk) Delete(ctx context.Context) error { if _, err := self.storage.zone.region.GetDisk(self.DiskId); err == cloudprovider.ErrNotFound { log.Errorf("Failed to find disk %s when delete", self.DiskId) return nil @@ -169,7 +170,7 @@ func (self *SDisk) Delete() error { return self.storage.zone.region.DeleteDisk(self.DiskId) } -func (self *SDisk) CreateISnapshot(name string, desc string) (cloudprovider.ICloudSnapshot, error) { +func (self *SDisk) CreateISnapshot(ctx context.Context, name string, desc string) (cloudprovider.ICloudSnapshot, error) { if snapshotId, err := self.storage.zone.region.CreateSnapshot(self.DiskId, name, desc); err != nil { log.Errorf("createSnapshot fail %s", err) return nil, err @@ -214,11 +215,11 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return isnapshots, nil } -func (self *SDisk) Resize(newSize int64) error { +func (self *SDisk) Resize(ctx context.Context, newSize int64) error { return self.storage.zone.region.resizeDisk(self.DiskId, newSize) } -func (self *SDisk) Reset(snapshotId string) error { +func (self *SDisk) Reset(ctx context.Context, snapshotId string) error { return self.storage.zone.region.resetDisk(self.DiskId, snapshotId) } diff --git a/pkg/util/aws/host.go b/pkg/util/aws/host.go index e6e3992b76..9f1782a931 100644 --- a/pkg/util/aws/host.go +++ b/pkg/util/aws/host.go @@ -268,3 +268,15 @@ func (self *SHost) _createVM(name, imgId string, sysDiskSize, cpu, memMB int, // 创建实例 return "", fmt.Errorf("Failed to create, specification not supported") } + +func (self *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (self *SHost) GetIsMaintenance() bool { + return false +} + +func (self *SHost) GetVersion() string { + return AWS_API_VERSION +} diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go index 4deb29516e..fe7c400710 100644 --- a/pkg/util/aws/instance.go +++ b/pkg/util/aws/instance.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "context" "github.com/aws/aws-sdk-go/service/ec2" "github.com/coredns/coredns/plugin/pkg/log" "yunion.io/x/jsonutils" @@ -299,7 +300,7 @@ func (self *SInstance) GetHypervisor() string { return models.HYPERVISOR_AWS } -func (self *SInstance) StartVM() error { +func (self *SInstance) StartVM(ctx context.Context) error { timeout := 300 * time.Second interval := 15 * time.Second @@ -323,7 +324,7 @@ func (self *SInstance) StartVM() error { return cloudprovider.ErrTimeout } -func (self *SInstance) StopVM(isForce bool) error { +func (self *SInstance) StopVM(ctx context.Context, isForce bool) error { err := self.host.zone.region.StopVM(self.InstanceId, isForce) if err != nil { return err @@ -331,7 +332,7 @@ func (self *SInstance) StopVM(isForce bool) error { return cloudprovider.WaitStatus(self, models.VM_READY, 10*time.Second, 300*time.Second) // 5mintues } -func (self *SInstance) DeleteVM() error { +func (self *SInstance) DeleteVM(ctx context.Context) error { for { err := self.host.zone.region.DeleteVM(self.InstanceId) if err != nil { @@ -344,16 +345,16 @@ func (self *SInstance) DeleteVM() error { } -func (self *SInstance) UpdateVM(name string) error { +func (self *SInstance) UpdateVM(ctx context.Context, name string) error { return self.host.zone.region.UpdateVM(self.InstanceId, name) } -func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { if len(publicKey) > 0 || len(passwd) > 0 { return "", fmt.Errorf("aws rebuild root not support specific publickey/password") } - diskId, err := self.host.zone.region.ReplaceSystemDisk(self.InstanceId, imageId, sysSizeGB) + diskId, err := self.host.zone.region.ReplaceSystemDisk(ctx, self.InstanceId, imageId, sysSizeGB) if err != nil { return "", err } @@ -361,11 +362,11 @@ func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey stri return diskId, nil } -func (self *SInstance) DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error { +func (self *SInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { return self.host.zone.region.DeployVM(self.InstanceId, name, password, publicKey, deleteKeypair, description) } -func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil) } @@ -373,7 +374,7 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { panic("implement me") } -func (self *SInstance) AttachDisk(diskId string) error { +func (self *SInstance) AttachDisk(ctx context.Context, diskId string) error { // todo：bugfix . self.DeviceNames => self.GetDeviceNames() name, err := NextDeviceName(self.DeviceNames) if err != nil { @@ -382,7 +383,7 @@ func (self *SInstance) AttachDisk(diskId string) error { return self.host.zone.region.AttachDisk(self.InstanceId, diskId, name) } -func (self *SInstance) DetachDisk(diskId string) error { +func (self *SInstance) DetachDisk(ctx context.Context, diskId string) error { return self.host.zone.region.DetachDisk(self.InstanceId, diskId) } @@ -713,7 +714,7 @@ func (self *SRegion) UpdateVM(instanceId string, hostname string) error { return fmt.Errorf("aws not support change hostname.") } -func (self *SRegion) ReplaceSystemDisk(instanceId string, imageId string, sysDiskSizeGB int) (string, error) { +func (self *SRegion) ReplaceSystemDisk(ctx context.Context, instanceId string, imageId string, sysDiskSizeGB int) (string, error) { instance, err := self.GetInstance(instanceId) if err != nil { return "", err @@ -747,13 +748,13 @@ func (self *SRegion) ReplaceSystemDisk(instanceId string, imageId string, sysDis self.ec2Client.WaitUntilVolumeAvailable(&ec2.DescribeVolumesInput{VolumeIds: []*string{&diskId}}) // todo: 检查instance状态 - err = instance.DetachDisk(rootDisk.DiskId) + err = instance.DetachDisk(ctx, rootDisk.DiskId) if err != nil { return "", err } self.ec2Client.WaitUntilInstanceStopped(&ec2.DescribeInstancesInput{InstanceIds: []*string{&instanceId}}) - err = instance.AttachDisk(diskId) + err = instance.AttachDisk(ctx, diskId) if err != nil { return "", err } diff --git a/pkg/util/aws/provider/provider.go b/pkg/util/aws/provider/provider.go index c774e51f6c..20c6327245 100644 --- a/pkg/util/aws/provider/provider.go +++ b/pkg/util/aws/provider/provider.go @@ -58,6 +58,10 @@ func (self *SAwsProvider) IsPublicCloud() bool { return true } +func (self *SAwsProvider) IsOnPremiseInfrastructure() bool { + return false +} + func (self *SAwsProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) { return self.client.GetIRegionById(id) } @@ -85,3 +89,7 @@ func (self *SAwsProvider) GetBalance() (float64, error) { } return balance.AvailableAmount, nil } + +func (self *SAwsProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/azure/classic_disk.go b/pkg/util/azure/classic_disk.go index 8946493c8e..3ead009503 100644 --- a/pkg/util/azure/classic_disk.go +++ b/pkg/util/azure/classic_disk.go @@ -4,6 +4,7 @@ import ( "strings" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -104,11 +105,11 @@ func (self *SClassicDisk) GetMetadata() *jsonutils.JSONDict { return nil } -func (self *SClassicDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) { +func (self *SClassicDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { return nil, cloudprovider.ErrNotSupported } -func (self *SClassicDisk) Delete() error { +func (self *SClassicDisk) Delete(ctx context.Context) error { return cloudprovider.ErrNotImplemented } @@ -184,8 +185,8 @@ func (self *SClassicDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error return nil, cloudprovider.ErrNotSupported } -func (self *SClassicDisk) GetIStorge() cloudprovider.ICloudStorage { - return self.storage +func (self *SClassicDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return self.storage, nil } func (self *SClassicDisk) GetName() string { @@ -204,10 +205,10 @@ func (self *SClassicDisk) Refresh() error { return nil } -func (self *SClassicDisk) Reset(snapshotId string) error { +func (self *SClassicDisk) Reset(ctx context.Context, snapshotId string) error { return cloudprovider.ErrNotSupported } -func (self *SClassicDisk) Resize(size int64) error { +func (self *SClassicDisk) Resize(ctx context.Context, size int64) error { return cloudprovider.ErrNotSupported } diff --git a/pkg/util/azure/classic_host.go b/pkg/util/azure/classic_host.go index 2679cc53cc..67833d2a21 100644 --- a/pkg/util/azure/classic_host.go +++ b/pkg/util/azure/classic_host.go @@ -171,3 +171,15 @@ func (self *SClassicHost) GetIWires() ([]cloudprovider.ICloudWire, error) { func (self *SClassicHost) GetManagerId() string { return self.zone.region.client.providerId } + +func (host *SClassicHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (host *SClassicHost) GetIsMaintenance() bool { + return false +} + +func (host *SClassicHost) GetVersion() string { + return AZURE_API_VERSION +} diff --git a/pkg/util/azure/classic_instance.go b/pkg/util/azure/classic_instance.go index 7753e39299..2b112464f9 100644 --- a/pkg/util/azure/classic_instance.go +++ b/pkg/util/azure/classic_instance.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -254,38 +255,38 @@ func (self *SClassicInstance) GetIHost() cloudprovider.ICloudHost { return self.host } -func (self *SClassicInstance) AttachDisk(diskId string) error { +func (self *SClassicInstance) AttachDisk(ctx context.Context, diskId string) error { if err := self.host.zone.region.AttachDisk(self.ID, diskId); err != nil { return err } return cloudprovider.WaitStatus(self, self.GetStatus(), 10*time.Second, 300*time.Second) } -func (self *SClassicInstance) DetachDisk(diskId string) error { +func (self *SClassicInstance) DetachDisk(ctx context.Context, diskId string) error { if err := self.host.zone.region.DetachDisk(self.ID, diskId); err != nil { return err } return cloudprovider.WaitStatus(self, self.GetStatus(), 10*time.Second, 300*time.Second) } -func (self *SClassicInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { - if err := self.host.zone.region.ChangeVMConfig(instanceId, ncpu, vmem); err != nil { +func (self *SClassicInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { + if err := self.host.zone.region.ChangeVMConfig(ctx, instanceId, ncpu, vmem); err != nil { return err } return cloudprovider.WaitStatus(self, self.GetStatus(), 10*time.Second, 300*time.Second) } -func (self *SClassicInstance) DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error { +func (self *SClassicInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { return cloudprovider.ErrNotImplemented //return self.host.zone.region.DeployVM(self.ID, name, password, publicKey, deleteKeypair, description) } -func (self *SClassicInstance) RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SClassicInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { return "", cloudprovider.ErrNotImplemented //return self.host.zone.region.ReplaceSystemDisk(self.ID, imageId, passwd, publicKey, int32(sysSizeGB)) } -func (self *SClassicInstance) UpdateVM(name string) error { +func (self *SClassicInstance) UpdateVM(ctx context.Context, name string) error { return cloudprovider.ErrNotSupported } @@ -301,7 +302,7 @@ func (self *SClassicInstance) GetGlobalId() string { return strings.ToLower(self.ID) } -func (self *SClassicInstance) DeleteVM() error { +func (self *SClassicInstance) DeleteVM(ctx context.Context) error { if err := self.host.zone.region.DeleteVM(self.ID); err != nil { return err } @@ -395,14 +396,14 @@ func (self *SClassicInstance) GetVNCInfo() (jsonutils.JSONObject, error) { return ret, nil } -func (self *SClassicInstance) StartVM() error { +func (self *SClassicInstance) StartVM(ctx context.Context) error { if err := self.host.zone.region.StartVM(self.ID); err != nil { return err } return cloudprovider.WaitStatus(self, models.VM_RUNNING, 10*time.Second, 300*time.Second) } -func (self *SClassicInstance) StopVM(isForce bool) error { +func (self *SClassicInstance) StopVM(ctx context.Context, isForce bool) error { err := self.host.zone.region.StopClassicVM(self.ID, isForce) if err != nil { return err diff --git a/pkg/util/azure/disk.go b/pkg/util/azure/disk.go index e1e8caa9ae..06cb70e2fb 100644 --- a/pkg/util/azure/disk.go +++ b/pkg/util/azure/disk.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -168,11 +169,11 @@ func (self *SDisk) Refresh() error { return nil } -func (self *SDisk) Delete() error { +func (self *SDisk) Delete(ctx context.Context) error { return self.storage.zone.region.deleteDisk(self.ID) } -func (self *SDisk) Resize(size int64) error { +func (self *SDisk) Resize(ctx context.Context, size int64) error { return self.storage.zone.region.ResizeDisk(self.ID, int32(size)) } @@ -191,8 +192,8 @@ func (self *SDisk) IsEmulated() bool { return false } -func (self *SDisk) GetIStorge() cloudprovider.ICloudStorage { - return self.storage +func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return self.storage, nil } func (self *SDisk) GetFsFormat() string { @@ -241,7 +242,7 @@ func (self *SDisk) GetDiskType() string { return models.DISK_TYPE_DATA } -func (self *SDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) { +func (self *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { if snapshot, err := self.storage.zone.region.CreateSnapshot(self.ID, name, desc); err != nil { log.Errorf("createSnapshot fail %s", err) return nil, err @@ -274,7 +275,7 @@ func (self *SDisk) GetBillingType() string { } func (self *SDisk) GetExpiredAt() time.Time { - return time.Now() + return time.Time{} } func (self *SDisk) GetSnapshotDetail(snapshotId string) (*SSnapshot, error) { @@ -313,7 +314,7 @@ func (region *SRegion) GetSnapShots(diskId string) ([]SSnapshot, error) { return result, nil } -func (self *SDisk) Reset(snapshotId string) error { +func (self *SDisk) Reset(ctx context.Context, snapshotId string) error { return self.storage.zone.region.resetDisk(self.ID, snapshotId) } diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go index 7472d84e93..f4373f6608 100644 --- a/pkg/util/azure/host.go +++ b/pkg/util/azure/host.go @@ -252,3 +252,15 @@ func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { func (self *SHost) GetManagerId() string { return self.zone.region.client.providerId } + +func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (host *SHost) GetIsMaintenance() bool { + return false +} + +func (host *SHost) GetVersion() string { + return AZURE_API_VERSION +} diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index c9aaf332d7..537b674c37 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -476,7 +477,7 @@ func (self *SInstance) GetIHost() cloudprovider.ICloudHost { return self.host } -func (self *SInstance) AttachDisk(diskId string) error { +func (self *SInstance) AttachDisk(ctx context.Context, diskId string) error { if err := self.host.zone.region.AttachDisk(self.ID, diskId); err != nil { return err } @@ -518,7 +519,7 @@ func (region *SRegion) AttachDisk(instanceId, diskId string) error { return region.client.Update(jsonutils.Marshal(instance), nil) } -func (self *SInstance) DetachDisk(diskId string) error { +func (self *SInstance) DetachDisk(ctx context.Context, diskId string) error { if err := self.host.zone.region.DetachDisk(self.ID, diskId); err != nil { return err } @@ -546,7 +547,7 @@ func (region *SRegion) DetachDisk(instanceId, diskId string) error { return region.client.Update(jsonutils.Marshal(instance), nil) } -func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { for _, vmSize := range self.host.zone.region.getHardwareProfile(ncpu, vmem) { self.Properties.HardwareProfile.VMSize = vmSize self.Properties.ProvisioningState = "" @@ -560,16 +561,16 @@ func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error return fmt.Errorf("Failed to change vm config, specification not supported") } -func (region *SRegion) ChangeVMConfig(instanceId string, ncpu int, vmem int) error { +func (region *SRegion) ChangeVMConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { instacen, err := region.GetInstance(instanceId) if err != nil { return err } - return instacen.ChangeConfig(instanceId, ncpu, vmem) + return instacen.ChangeConfig(ctx, instanceId, ncpu, vmem) } -func (self *SInstance) DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error { - return self.host.zone.region.DeployVM(self.ID, name, password, publicKey, deleteKeypair, description) +func (self *SInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { + return self.host.zone.region.DeployVM(ctx, self.ID, name, password, publicKey, deleteKeypair, description) } type VirtualMachineExtensionProperties struct { @@ -659,7 +660,7 @@ func (region *SRegion) resetPassword(instanceId, username, password string) erro return region.resetLoginInfo(instanceId, setting) } -func (region *SRegion) DeployVM(instanceId, name, password, publicKey string, deleteKeypair bool, description string) error { +func (region *SRegion) DeployVM(ctx context.Context, instanceId, name, password, publicKey string, deleteKeypair bool, description string) error { instance, err := region.GetInstance(instanceId) if err != nil { return err @@ -673,7 +674,7 @@ func (region *SRegion) DeployVM(instanceId, name, password, publicKey string, de return region.resetPassword(instanceId, instance.Properties.OsProfile.AdminUsername, password) } -func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { return self.host.zone.region.ReplaceSystemDisk(self.ID, imageId, passwd, publicKey, int32(sysSizeGB)) } @@ -759,7 +760,7 @@ func (region *SRegion) ReplaceSystemDisk(instanceId, imageId, passwd, publicKey return disk.ID, nil } -func (self *SInstance) UpdateVM(name string) error { +func (self *SInstance) UpdateVM(ctx context.Context, name string) error { return cloudprovider.ErrNotSupported } @@ -779,7 +780,7 @@ func (self *SRegion) DeleteVM(instanceId string) error { return self.doDeleteVM(instanceId) } -func (self *SInstance) DeleteVM() error { +func (self *SInstance) DeleteVM(ctx context.Context) error { sysDiskId := "" if self.Properties.StorageProfile.OsDisk.ManagedDisk != nil { sysDiskId = self.Properties.StorageProfile.OsDisk.ManagedDisk.ID @@ -960,7 +961,7 @@ func (self *SRegion) StartVM(instanceId string) error { return err } -func (self *SInstance) StartVM() error { +func (self *SInstance) StartVM(ctx context.Context) error { if err := self.host.zone.region.StartVM(self.ID); err != nil { return err } @@ -968,7 +969,7 @@ func (self *SInstance) StartVM() error { return cloudprovider.WaitStatus(self, models.VM_RUNNING, 10*time.Second, 300*time.Second) } -func (self *SInstance) StopVM(isForce bool) error { +func (self *SInstance) StopVM(ctx context.Context, isForce bool) error { err := self.host.zone.region.StopVM(self.ID, isForce) if err != nil { return err @@ -1033,7 +1034,7 @@ func (self *SInstance) GetBillingType() string { } func (self *SInstance) GetExpiredAt() time.Time { - return time.Now() + return time.Time{} } func (self *SInstance) UpdateUserData(userData string) error { diff --git a/pkg/util/azure/provider/provider.go b/pkg/util/azure/provider/provider.go index 5430910e52..214877edd3 100644 --- a/pkg/util/azure/provider/provider.go +++ b/pkg/util/azure/provider/provider.go @@ -35,6 +35,10 @@ func (self *SAzureProvider) IsPublicCloud() bool { return true } +func (self *SAzureProvider) IsOnPremiseInfrastructure() bool { + return false +} + func (self *SAzureProvider) GetId() string { return azure.CLOUD_PROVIDER_AZURE } @@ -86,3 +90,7 @@ func (self *SAzureProvider) GetBalance() (float64, error) { } return balance.AvailableAmount, nil } + +func (self *SAzureProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/esxi/datacenter.go b/pkg/util/esxi/datacenter.go index 7da30e1dfb..652facd020 100644 --- a/pkg/util/esxi/datacenter.go +++ b/pkg/util/esxi/datacenter.go @@ -76,3 +76,29 @@ func (dc *SDatacenter) GetIStorages() ([]cloudprovider.ICloudStorage, error) { } return dc.istorages, nil } + +func (dc *SDatacenter) GetIHostByMoId(idstr string) (cloudprovider.ICloudHost, error) { + ihosts, err := dc.GetIHosts() + if err != nil { + return nil, err + } + for i := 0; i < len(ihosts); i += 1 { + if ihosts[i].GetId() == idstr { + return ihosts[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (dc *SDatacenter) GetIStorageByMoId(idstr string) (cloudprovider.ICloudStorage, error) { + istorages, err := dc.GetIStorages() + if err != nil { + return nil, err + } + for i := 0; i < len(istorages); i += 1 { + if istorages[i].GetId() == idstr { + return istorages[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} diff --git a/pkg/util/esxi/device.go b/pkg/util/esxi/device.go new file mode 100644 index 0000000000..4e2df97367 --- /dev/null +++ b/pkg/util/esxi/device.go @@ -0,0 +1,92 @@ +package esxi + +import ( + "reflect" + "strings" + + "github.com/vmware/govmomi/vim25/types" + // "yunion.io/x/log" +) + +type SVirtualDevice struct { + vm *SVirtualMachine + dev types.BaseVirtualDevice + index int +} + +func NewVirtualDevice(vm *SVirtualMachine, dev types.BaseVirtualDevice, index int) SVirtualDevice { + return SVirtualDevice{ + vm: vm, + dev: dev, + index: index, + } +} + +func (dev *SVirtualDevice) getKey() int32 { + return dev.dev.GetVirtualDevice().Key +} + +func (dev *SVirtualDevice) getControllerKey() int32 { + return dev.dev.GetVirtualDevice().ControllerKey +} + +func (dev *SVirtualDevice) getIndex() int { + return dev.index +} + +func (dev *SVirtualDevice) getLabel() string { + return dev.dev.GetVirtualDevice().DeviceInfo.GetDescription().Label +} + +func (dev *SVirtualDevice) GetDriver() string { + val := reflect.Indirect(reflect.ValueOf(dev.dev)) + driver := strings.ToLower(val.Type().Name()) + if strings.Contains(driver, "virtualmachine") { + return strings.Replace(driver, "virtualmachine", "", -1) + } else if strings.Contains(driver, "virtual") { + return strings.Replace(driver, "virtual", "", -1) + } else { + return driver + } +} + +type SVirtualVGA struct { + SVirtualDevice +} + +func NewVirtualVGA(vm *SVirtualMachine, dev types.BaseVirtualDevice, index int) SVirtualVGA { + return SVirtualVGA{ + NewVirtualDevice(vm, dev, index), + } +} + +func (vga *SVirtualVGA) getVirtualMachineVideoCard() *types.VirtualMachineVideoCard { + return vga.dev.(*types.VirtualMachineVideoCard) +} + +func (vga *SVirtualVGA) GetEnable3D() bool { + p3d := vga.getVirtualMachineVideoCard().Enable3DSupport + return p3d != nil && *p3d +} + +func (vga *SVirtualVGA) GetRamSizeMB() int { + return int(vga.getVirtualMachineVideoCard().VideoRamSizeInKB / 1024) +} + +func (vga *SVirtualVGA) String() string { + return vga.getVirtualMachineVideoCard().DeviceInfo.GetDescription().Summary +} + +type SVirtualCdrom struct { + SVirtualDevice +} + +func NewVirtualCdrom(vm *SVirtualMachine, dev types.BaseVirtualDevice, index int) SVirtualCdrom { + return SVirtualCdrom{ + NewVirtualDevice(vm, dev, index), + } +} + +func (cdrom *SVirtualCdrom) getVirtualCdrom() *types.VirtualCdrom { + return cdrom.dev.(*types.VirtualCdrom) +} diff --git a/pkg/util/esxi/host.go b/pkg/util/esxi/host.go index 2ab2bd5350..fc4581fb9e 100644 --- a/pkg/util/esxi/host.go +++ b/pkg/util/esxi/host.go @@ -5,32 +5,23 @@ import ( "yunion.io/x/jsonutils" + "fmt" "github.com/vmware/govmomi/vim25/types" + "strings" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/pkg/util/netutils" ) -var HOST_SYSTEM_PROPS = []string{"name", "parent", "summary", "config", "hardware", "vm"} - -type SHostNicInfo struct { - Dev string - Driver string - Mac string - Index int - LinkUp bool - IpAddr string - Mtu int - NicType string -} +var HOST_SYSTEM_PROPS = []string{"name", "parent", "summary", "config", "hardware", "vm", "datastore"} type SHostStorageAdapterInfo struct { Device string Model string Driver string Pci string - Drivers []SHostStorageDriverInfo + Drivers []*SHostStorageDriverInfo Enclosure int } @@ -44,6 +35,7 @@ type SHostStorageDriverInfo struct { SSD bool Dev string Size int + Slot int } type SHostStorageEnclosureInfo struct { @@ -55,11 +47,23 @@ type SHostStorageEnclosureInfo struct { Status string } +type SHostStorageInfo struct { + Adapter int + Driver string + Index int + Model string + Rotate bool + Status string + Size int +} + type SHost struct { SManagedObject nicInfo []SHostNicInfo - storageInfo []SHostStorageAdapterInfo + storageInfo []SHostStorageInfo + + datastores []cloudprovider.ICloudStorage vms []cloudprovider.ICloudVM } @@ -98,7 +102,8 @@ func (self *SHost) GetStatus() string { } func (self *SHost) Refresh() error { - return cloudprovider.ErrNotImplemented + self.vms = nil + return nil } func (self *SHost) IsEmulated() bool { @@ -155,11 +160,29 @@ func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { } func (self *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) { - return nil, cloudprovider.ErrNotImplemented + moHost := self.getHostSystem() + istorages := make([]cloudprovider.ICloudStorage, len(moHost.Datastore)) + for i := 0; i < len(moHost.Datastore); i += 1 { + storage, err := self.datacenter.GetIStorageByMoId(moRefId(moHost.Datastore[i])) + if err != nil { + return nil, err + } + istorages[i] = storage + } + return istorages, nil } func (self *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return nil, cloudprovider.ErrNotImplemented + istorages, err := self.GetIStorages() + if err != nil { + return nil, err + } + for i := 0; i < len(istorages); i += 1 { + if istorages[i].GetGlobalId() == id { + return istorages[i], nil + } + } + return nil, cloudprovider.ErrNotFound } func (self *SHost) GetEnabled() bool { @@ -221,7 +244,7 @@ func (self *SHost) fetchNicInfo() []SHostNicInfo { info.Dev = nic.Device info.Driver = nic.Driver info.Mac = netutils.FormatMacAddr(nic.Mac) - info.Index = i + info.Index = int8(i) info.LinkUp = false nicInfoList = append(nicInfoList, info) } @@ -295,58 +318,150 @@ func (self *SHost) GetMemSizeMB() int { return int(self.getHostSystem().Summary.Hardware.MemorySize / 1024 / 1024) } -/*func (self *SHost) fetchStorageInfo() { - adapterList := make([]SHostStorageAdapterInfo, 0) - driversTable := make(map[string]SHostStorageDriverInfo, 0) - enclosuresTable := make(map[string]SHostStorageEnclosureInfo, 0) +func (self *SHost) GetStorageInfo() []SHostStorageInfo { + if self.storageInfo == nil { + self.storageInfo = self.getStorageInfo() + } + return self.storageInfo +} + +func (self *SHost) getStorageInfo() []SHostStorageInfo { + diskSlots := make(map[int]SHostStorageInfo) + list := self.getStorages() + for i := 0; i < len(list); i += 1 { + for j := 0; j < len(list[i].Drivers); j += 1 { + drv := list[i].Drivers[j] + info := SHostStorageInfo{ + Adapter: 0, + Driver: "Linux", + Index: drv.Slot, + Model: strings.TrimSpace(fmt.Sprintf("%s %s", drv.Vendor, drv.Model)), + Rotate: !drv.SSD, + Status: drv.Status, + Size: drv.Size, + } + diskSlots[info.Index] = info + } + } + disks := make([]SHostStorageInfo, 0) + idx := 0 + for { + if info, ok := diskSlots[idx]; ok { + disks = append(disks, info) + idx += 1 + } else { + break + } + } + return disks +} + +func (self *SHost) getStorages() []*SHostStorageAdapterInfo { + adapterList := make([]*SHostStorageAdapterInfo, 0) + adapterTable := make(map[string]*SHostStorageAdapterInfo) + driversTable := make(map[string]*SHostStorageDriverInfo, 0) + enclosuresTable := make(map[string]*SHostStorageEnclosureInfo, 0) moHost := self.getHostSystem() - for i, ad := range moHost.Config.StorageDevice.HostBusAdapter { + for i := 0; i < len(moHost.Config.StorageDevice.HostBusAdapter); i += 1 { + ad := moHost.Config.StorageDevice.HostBusAdapter[i] adinfo := ad.GetHostHostBusAdapter() if adinfo == nil { - log.Errorf("Fail to GetHostHostBusAdapter") + log.Errorf("fail to GetHostHostBusAdapter") continue } info := SHostStorageAdapterInfo{} info.Device = adinfo.Device - info.Model = adinfo.Model + info.Model = strings.TrimSpace(adinfo.Model) info.Driver = adinfo.Driver info.Pci = adinfo.Pci - info.Drivers = make([]SHostStorageDriverInfo, 0) + info.Drivers = make([]*SHostStorageDriverInfo, 0) info.Enclosure = -1 - adapterList = append(adapterList, info) + + adapterTable[adinfo.Key] = &info + adapterList = append(adapterList, &info) } - for i, drv := range moHost.Config.StorageDevice.ScsiLun { + for i := 0; i < len(moHost.Config.StorageDevice.ScsiLun); i += 1 { + drv := moHost.Config.StorageDevice.ScsiLun[i] lunInfo := drv.GetScsiLun() if lunInfo == nil { log.Errorf("fail to GetScsiLun") continue } - if lunInfo.DeviceType == "disk" { - info := SHostStorageDriverInfo{} - info.CN = lunInfo.CanonicalName - info.Name = lunInfo.DisplayName - info.Model = lunInfo.Model - info.Vendor = lunInfo.Vendor - info.Revision = lunInfo.Revision - info.Status = lunInfo.OperationalState[0] - // info.SSD = lunInfo. - // info.Dev = - // info.Size = lunInfo.S - } else if lunInfo.DeviceType == "enclosure" { + if lunInfo.DeviceType == "disk" { + scsiDisk := drv.(*types.HostScsiDisk) + info := SHostStorageDriverInfo{} + info.CN = scsiDisk.CanonicalName + info.Name = scsiDisk.DisplayName + info.Model = strings.TrimSpace(scsiDisk.Model) + info.Vendor = strings.TrimSpace(scsiDisk.Vendor) + info.Revision = scsiDisk.Revision + info.Status = scsiDisk.OperationalState[0] + if scsiDisk.Ssd != nil && *scsiDisk.Ssd { + info.SSD = true + } + info.Dev = scsiDisk.DevicePath + info.Size = int(int64(scsiDisk.Capacity.BlockSize) * scsiDisk.Capacity.Block / 1024 / 1024) + + driversTable[scsiDisk.Key] = &info + } else if lunInfo.DeviceType == "enclosure" { + enclosuresTable[lunInfo.Key] = &SHostStorageEnclosureInfo{ + CN: lunInfo.CanonicalName, + Name: lunInfo.DisplayName, + Model: strings.TrimSpace(lunInfo.Model), + Vendor: strings.TrimSpace(lunInfo.Vendor), + Revision: lunInfo.Revision, + Status: lunInfo.OperationalState[0], + } } } + for i := 0; i < len(moHost.Config.StorageDevice.ScsiTopology.Adapter); i += 1 { + ad := moHost.Config.StorageDevice.ScsiTopology.Adapter[i] + adapter := adapterTable[ad.Adapter] + for j := 0; j < len(ad.Target); j += 1 { + t := ad.Target[j] + key := t.Lun[0].ScsiLun + if _, ok := enclosuresTable[key]; ok { + adapter.Enclosure = int(t.Target) + } else if _, ok := driversTable[key]; ok { + driver := driversTable[key] + driver.Slot = int(t.Target) + adapter.Drivers = append(adapter.Drivers, driver) + } + } + } + return adapterList } -*/ func (self *SHost) GetStorageSizeMB() int { - return 0 + size := 0 + storages := self.GetStorageInfo() + for i := 0; i < len(storages); i += 1 { + size += storages[i].Size + } + return size } func (self *SHost) GetStorageType() string { - return "" + ssd := 0 + rotate := 0 + storages := self.GetStorageInfo() + for i := 0; i < len(storages); i += 1 { + if storages[i].Rotate { + rotate += 1 + } else { + ssd += 1 + } + } + if ssd == 0 && rotate > 0 { + return models.DISK_TYPE_ROTATE + } else if ssd > 0 && rotate == 0 { + return models.DISK_TYPE_SSD + } else { + return models.DISK_TYPE_HYBRID + } } func (self *SHost) GetHostType() string { @@ -357,8 +472,28 @@ func (self *SHost) GetManagerId() string { return self.manager.providerId } +func (self *SHost) GetIsMaintenance() bool { + moHost := self.getHostSystem() + return moHost.Summary.Runtime.InMaintenanceMode +} + +func (self *SHost) GetVersion() string { + moHost := self.getHostSystem() + about := moHost.Summary.Config.Product + return fmt.Sprintf("%s-%s", about.Version, about.Build) +} + func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, memMB int, vswitchId string, ipAddr string, desc string, passwd string, storageType string, diskSizes []int, publicKey string, secGrpId string, userData string) (cloudprovider.ICloudVM, error) { log.Debugf("CreateVM") return nil, cloudprovider.ErrNotImplemented } + +func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + nics := host.getNicInfo() + inics := make([]cloudprovider.ICloudHostNetInterface, len(nics)) + for i := 0; i < len(nics); i += 1 { + inics[i] = &nics[i] + } + return inics, nil +} diff --git a/pkg/util/esxi/hostnic.go b/pkg/util/esxi/hostnic.go new file mode 100644 index 0000000000..242c7ae76b --- /dev/null +++ b/pkg/util/esxi/hostnic.go @@ -0,0 +1,44 @@ +package esxi + +type SHostNicInfo struct { + Dev string + Driver string + Mac string + Index int8 + LinkUp bool + IpAddr string + Mtu int16 + NicType string +} + +func (nic *SHostNicInfo) GetDevice() string { + return nic.Dev +} + +func (nic *SHostNicInfo) GetDriver() string { + return nic.Driver +} + +func (nic *SHostNicInfo) GetMac() string { + return nic.Mac +} + +func (nic *SHostNicInfo) GetIndex() int8 { + return nic.Index +} + +func (nic *SHostNicInfo) IsLinkUp() bool { + return nic.LinkUp +} + +func (nic *SHostNicInfo) GetIpAddr() string { + return nic.IpAddr +} + +func (nic *SHostNicInfo) GetMtu() int16 { + return nic.Mtu +} + +func (nic *SHostNicInfo) GetNicType() string { + return nic.NicType +} diff --git a/pkg/util/esxi/manager.go b/pkg/util/esxi/manager.go index 678d8510e7..9b1f83da1b 100644 --- a/pkg/util/esxi/manager.go +++ b/pkg/util/esxi/manager.go @@ -50,14 +50,18 @@ func NewESXiClient(providerId string, providerName string, host string, port int return cli, nil } -func (cli *SESXiClient) url() string { +func (cli *SESXiClient) getUrl() string { if cli.port == 443 || cli.port == 0 { - return fmt.Sprintf("https://%s/sdk", cli.host) + return fmt.Sprintf("https://%s", cli.host) } else { - return fmt.Sprintf("https://%s:%d/sdk", cli.host, cli.port) + return fmt.Sprintf("https://%s:%d", cli.host, cli.port) } } +func (cli *SESXiClient) url() string { + return fmt.Sprintf("%s/sdk", cli.getUrl()) +} + func (cli *SESXiClient) connect() error { u, err := url.Parse(cli.url()) if err != nil { @@ -101,7 +105,18 @@ func (cli *SESXiClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { } func (cli *SESXiClient) About() jsonutils.JSONObject { - return jsonutils.Marshal(&cli.client.ServiceContent.About) + about := jsonutils.Marshal(&cli.client.ServiceContent.About) + aboutDict := about.(*jsonutils.JSONDict) + aboutDict.Add(jsonutils.NewString(cli.getEndpointType()), "endpoint_type") + return aboutDict +} + +func (cli *SESXiClient) getEndpointType() string { + if cli.IsVCenter() { + return "VCenter" + } else { + return "ESXi" + } } func (cli *SESXiClient) GetUUID() string { @@ -118,6 +133,16 @@ func (cli *SESXiClient) fetchDatacenters() error { cli.datacenters = make([]*SDatacenter, len(dcs)) for i := 0; i < len(dcs); i += 1 { cli.datacenters[i] = newDatacenter(cli, &dcs[i]) + + err = cli.datacenters[i].scanHosts() + if err != nil { + return err + } + + err = cli.datacenters[i].scanDatastores() + if err != nil { + return err + } } return nil } @@ -151,36 +176,9 @@ func (cli *SESXiClient) scanMObjects(folder types.ManagedObjectReference, props return nil } -/* -func getStructFields(dst interface{}) []string { - dataValue := reflect.Indirect(reflect.ValueOf(dst)) - dataType := dataValue.Type() - if dataType.Kind() != reflect.Struct { - log.Warningf("GetStructFeilds for non-struct data") - return nil - } - return _getStructFields(dataType) -} - -func _getStructFields(dataType reflect.Type) []string { - ret := make([]string, 0) - for i := 1; i < dataType.NumField(); i += 1 { - field := dataType.Field(i) - if field.Type.Kind() == reflect.Struct && field.Anonymous { - subfields := _getStructFields(field.Type) - ret = append(ret, subfields...) - } else if gotypes.IsFieldExportable(field.Name) { - log.Debugf("%s: %s", field.Name, field.Type.Name()) - ret = append(ret, utils.CamelSplit(field.Name, "_")) - } - } - return ret -} -*/ - func (cli *SESXiClient) references2Objects(refs []types.ManagedObjectReference, props []string, dst interface{}) error { pc := property.DefaultCollector(cli.client.Client) - err := pc.Retrieve(cli.context, refs, []string{"name", "config", "summary"}, dst) + err := pc.Retrieve(cli.context, refs, props, dst) if err != nil { return err } @@ -202,7 +200,7 @@ func (cli *SESXiClient) GetDatacenters() ([]*SDatacenter, error) { return cli.datacenters, nil } -func (cli *SESXiClient) FindDatacenterById(dcId string) (*SDatacenter, error) { +func (cli *SESXiClient) FindDatacenterByMoId(dcId string) (*SDatacenter, error) { dcs, err := cli.GetDatacenters() if err != nil { return nil, err @@ -215,8 +213,22 @@ func (cli *SESXiClient) FindDatacenterById(dcId string) (*SDatacenter, error) { return nil, cloudprovider.ErrNotFound } +func (cli *SESXiClient) FindHostByMoId(moId string) (cloudprovider.ICloudHost, error) { + dcs, err := cli.GetDatacenters() + if err != nil { + return nil, err + } + for i := 0; i < len(dcs); i += 1 { + ihost, err := dcs[i].GetIHostByMoId(moId) + if err == nil { + return ihost, nil + } + } + return nil, cloudprovider.ErrNotFound +} + func (cli *SESXiClient) getPrivateId(idStr string) string { - if strings.HasPrefix(idStr, cli.providerId) { + if len(cli.providerId) > 0 && strings.HasPrefix(idStr, cli.providerId) { idStr = idStr[len(cli.providerId)+1:] } return idStr @@ -224,12 +236,17 @@ func (cli *SESXiClient) getPrivateId(idStr string) string { func (cli *SESXiClient) FindHostByIp(hostIp string) (*SHost, error) { searchIndex := object.NewSearchIndex(cli.client.Client) + hostRef, err := searchIndex.FindByIp(cli.context, nil, cli.getPrivateId(hostIp), false) if err != nil { log.Errorf("searchIndex.FindByIp fail %s", err) return nil, err } + if hostRef == nil { + return nil, fmt.Errorf("cannot find %s", cli.getPrivateId(hostIp)) + } + var host mo.HostSystem err = cli.reference2Object(hostRef.Reference(), HOST_SYSTEM_PROPS, &host) if err != nil { @@ -244,3 +261,11 @@ func (cli *SESXiClient) acquireCloneTicket() (string, error) { manager := session.NewManager(cli.client.Client) return manager.AcquireCloneTicket(cli.context) } + +func (cli *SESXiClient) IsVCenter() bool { + return cli.client.Client.IsVC() +} + +func (cli *SESXiClient) IsValid() bool { + return cli.client.Client.Valid() +} diff --git a/pkg/util/esxi/mobase.go b/pkg/util/esxi/mobase.go index c60498ebc5..fe8a71d0d6 100644 --- a/pkg/util/esxi/mobase.go +++ b/pkg/util/esxi/mobase.go @@ -1,7 +1,7 @@ package esxi import ( - "reflect" + "strings" "github.com/vmware/govmomi/vim25/mo" @@ -27,11 +27,11 @@ func (self *SManagedObject) GetName() string { } func (self *SManagedObject) GetId() string { - return self.object.Entity().Self.Value + return moRefId(self.object.Entity().Self) } func (self *SManagedObject) GetType() string { - return self.object.Entity().Self.Type + return moRefType(self.object.Entity().Self) } func (self *SManagedObject) getCurrentParentEntity() *mo.ManagedEntity { @@ -47,28 +47,12 @@ func (self *SManagedObject) getParentEntity(obj *mo.ManagedEntity) *mo.ManagedEn log.Errorf("%s", err) return nil } - log.Debugf("getParentEntity %s %s %s", entity.Self.Type, entity.Self.Value, entity.Name) + // log.Debugf("getParentEntity %s %s %s", entity.Self.Type, entity.Self.Value, entity.Name) return &entity } return nil } -func reverseArray(array interface{}) { - arrayValue := reflect.Indirect(reflect.ValueOf(array)) - if arrayValue.Kind() != reflect.Slice && arrayValue.Kind() != reflect.Array { - log.Errorf("reverse non array or slice") - return - } - tmp := reflect.Indirect(reflect.New(arrayValue.Type().Elem())) - for i, j := 0, arrayValue.Len()-1; i < j; i, j = i+1, j-1 { - tmpi := arrayValue.Index(i) - tmpj := arrayValue.Index(j) - tmp.Set(tmpi) - tmpi.Set(tmpj) - tmpj.Set(tmp) - } -} - func (self *SManagedObject) fetchPath() []string { path := make([]string, 0) obj := self.object.Entity() @@ -91,6 +75,7 @@ func (self *SManagedObject) findInParents(objType string) *mo.ManagedEntity { obj := self.object.Entity() for obj != nil && obj.Self.Type != objType { + // log.Debugf("find %s want %s", obj.Self.Type, objType) obj = self.getParentEntity(obj) } @@ -102,7 +87,7 @@ func (self *SManagedObject) fetchDatacenter() (*SDatacenter, error) { if me == nil { return nil, cloudprovider.ErrNotFound } - return self.manager.FindDatacenterById(me.Self.Value) + return self.manager.FindDatacenterByMoId(me.Self.Value) } func (self *SManagedObject) GetDatacenter() (*SDatacenter, error) { @@ -125,3 +110,15 @@ func (self *SManagedObject) GetDatacenterPath() []string { path := dc.GetPath() return path[1:] } + +func (self *SManagedObject) GetDatacenterPathString() string { + path := self.GetDatacenterPath() + if path != nil { + return strings.Join(path, "/") + } + return "" +} + +func (self *SManagedObject) getManagerUri() string { + return self.manager.getUrl() +} diff --git a/pkg/util/esxi/provider/provider.go b/pkg/util/esxi/provider/provider.go index 8b292054f9..de481a0188 100644 --- a/pkg/util/esxi/provider/provider.go +++ b/pkg/util/esxi/provider/provider.go @@ -75,6 +75,10 @@ func (self *SESXiProvider) IsPublicCloud() bool { return false } +func (self *SESXiProvider) IsOnPremiseInfrastructure() bool { + return true +} + func (self *SESXiProvider) GetId() string { return esxi.CLOUD_PROVIDER_VMWARE } @@ -113,13 +117,30 @@ func (self *SESXiProvider) GetIVpcById(id string) (cloudprovider.ICloudVpc, erro } func (self *SESXiProvider) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return nil, cloudprovider.ErrNotImplemented + return nil, cloudprovider.ErrNotSupported } func (self *SESXiProvider) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - return nil, cloudprovider.ErrNotImplemented + return nil, cloudprovider.ErrNotSupported } func (self *SESXiProvider) GetBalance() (float64, error) { return 0.0, nil } + +func (self *SESXiProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { + dcs, err := self.client.GetDatacenters() + if err != nil { + return nil, err + } + + ihosts := make([]cloudprovider.ICloudHost, 0) + for i := 0; i < len(dcs); i += 1 { + dcIHosts, err := dcs[i].GetIHosts() + if err != nil { + return nil, err + } + ihosts = append(ihosts, dcIHosts...) + } + return ihosts, nil +} diff --git a/pkg/util/esxi/shell/host.go b/pkg/util/esxi/shell/host.go index 2807ab0607..86deb3792d 100644 --- a/pkg/util/esxi/shell/host.go +++ b/pkg/util/esxi/shell/host.go @@ -10,7 +10,7 @@ func init() { DATACENTER string `help:"List hosts in datacenter"` } shellutils.R(&HostListOptions{}, "host-list", "List hosts in datacenter", func(cli *esxi.SESXiClient, args *HostListOptions) error { - dc, err := cli.FindDatacenterById(args.DATACENTER) + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) if err != nil { return err } diff --git a/pkg/util/esxi/shell/store.go b/pkg/util/esxi/shell/store.go index 6dcd6e6a7f..ec5efa21e2 100644 --- a/pkg/util/esxi/shell/store.go +++ b/pkg/util/esxi/shell/store.go @@ -1,7 +1,10 @@ package shell import ( + "context" + "os" "yunion.io/x/onecloud/pkg/util/esxi" + "yunion.io/x/onecloud/pkg/util/printutils" "yunion.io/x/onecloud/pkg/util/shellutils" ) @@ -10,7 +13,7 @@ func init() { DATACENTER string `help:"List datastores in datacenter"` } shellutils.R(&DatastoreListOptions{}, "ds-list", "List datastores in datacenter", func(cli *esxi.SESXiClient, args *DatastoreListOptions) error { - dc, err := cli.FindDatacenterById(args.DATACENTER) + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) if err != nil { return err } @@ -21,4 +24,141 @@ func init() { printList(ds, nil) return nil }) + + type DatastoreShowOptions struct { + DATACENTER string `help:"Datacenter"` + DSID string `help:"Datastore ID""` + } + shellutils.R(&DatastoreShowOptions{}, "ds-show", "Show details of a datastore", func(cli *esxi.SESXiClient, args *DatastoreShowOptions) error { + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) + if err != nil { + return err + } + ds, err := dc.GetIStorageByMoId(args.DSID) + if err != nil { + return err + } + printObject(ds) + return nil + }) + + type DatastoreListDirOptions struct { + DATACENTER string `help:"Datacenter"` + DSID string `help:"Datastore ID""` + DIR string `help:"directory"` + } + shellutils.R(&DatastoreListDirOptions{}, "ds-list-dir", "List directory of a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) + if err != nil { + return err + } + ds, err := dc.GetIStorageByMoId(args.DSID) + if err != nil { + return err + } + ctx := context.Background() + dsObj := ds.(*esxi.SDatastore) + fileList, err := dsObj.ListDir(ctx, args.DIR) + if err != nil { + return err + } + printutils.PrintInterfaceList(fileList, 0, 0, 0, []string{"Name", "Date", "Size"}) + return nil + }) + + shellutils.R(&DatastoreListDirOptions{}, "ds-check-file", "Check file status in a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) + if err != nil { + return err + } + ds, err := dc.GetIStorageByMoId(args.DSID) + if err != nil { + return err + } + ctx := context.Background() + dsObj := ds.(*esxi.SDatastore) + file, err := dsObj.CheckFile(ctx, args.DIR) + if err != nil { + return err + } + printutils.PrintInterfaceObject(file) + return nil + }) + + shellutils.R(&DatastoreListDirOptions{}, "ds-delete-file", "Delete file in a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) + if err != nil { + return err + } + ds, err := dc.GetIStorageByMoId(args.DSID) + if err != nil { + return err + } + ctx := context.Background() + dsObj := ds.(*esxi.SDatastore) + err = dsObj.Delete(ctx, args.DIR) + if err != nil { + return err + } + return nil + }) + + type DatastoreDownloadOptions struct { + DATACENTER string `help:"Datacenter"` + DSID string `help:"Datastore ID""` + DIR string `help:"directory"` + LOCAL string `help:"local file"` + } + shellutils.R(&DatastoreDownloadOptions{}, "ds-download", "Download file from a datastore", func(cli *esxi.SESXiClient, args *DatastoreDownloadOptions) error { + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) + if err != nil { + return err + } + ds, err := dc.GetIStorageByMoId(args.DSID) + if err != nil { + return err + } + ctx := context.Background() + dsObj := ds.(*esxi.SDatastore) + + file, err := os.Create(args.LOCAL) + if err != nil { + return err + } + defer file.Close() + + err = dsObj.Download(ctx, args.DIR, file) + if err != nil { + return err + } + + return nil + }) + + shellutils.R(&DatastoreDownloadOptions{}, "ds-upload", "Upload local file to datastore", func(cli *esxi.SESXiClient, args *DatastoreDownloadOptions) error { + dc, err := cli.FindDatacenterByMoId(args.DATACENTER) + if err != nil { + return err + } + ds, err := dc.GetIStorageByMoId(args.DSID) + if err != nil { + return err + } + ctx := context.Background() + dsObj := ds.(*esxi.SDatastore) + + file, err := os.Open(args.LOCAL) + if err != nil { + return err + } + defer file.Close() + + err = dsObj.Upload(ctx, args.DIR, file) + if err != nil { + return err + } + + return nil + }) + } diff --git a/pkg/util/esxi/shell/virtualmachine.go b/pkg/util/esxi/shell/virtualmachine.go index e1c9519054..4d1ace464b 100644 --- a/pkg/util/esxi/shell/virtualmachine.go +++ b/pkg/util/esxi/shell/virtualmachine.go @@ -40,6 +40,40 @@ func init() { return nil }) + shellutils.R(&VirtualMachineShowOptions{}, "vm-nics", "Show vm nics details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error { + host, err := cli.FindHostByIp(args.HOSTIP) + if err != nil { + return err + } + vm, err := host.GetIVMById(args.VMID) + if err != nil { + return err + } + vmnics, err := vm.GetINics() + if err != nil { + return err + } + printList(vmnics, []string{}) + return nil + }) + + shellutils.R(&VirtualMachineShowOptions{}, "vm-disks", "Show vm disks details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error { + host, err := cli.FindHostByIp(args.HOSTIP) + if err != nil { + return err + } + vm, err := host.GetIVMById(args.VMID) + if err != nil { + return err + } + vmdisks, err := vm.GetIDisks() + if err != nil { + return err + } + printList(vmdisks, []string{}) + return nil + }) + shellutils.R(&VirtualMachineShowOptions{}, "vm-vnc", "Show vm VNC details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error { host, err := cli.FindHostByIp(args.HOSTIP) if err != nil { @@ -56,4 +90,5 @@ func init() { printutils.PrintJSONObject(info) return nil }) + } diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index b64c57ce1f..0b0b2576d2 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -3,15 +3,31 @@ package esxi import ( "github.com/vmware/govmomi/vim25/mo" + "context" + "fmt" + "github.com/vmware/govmomi/vim25/types" + "io" + "io/ioutil" + "net/http" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "time" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/vmdkutils" ) -var DATASTORE_PROPS = []string{"name", "parent"} +var DATASTORE_PROPS = []string{"name", "parent", "info", "summary", "host"} type SDatastore struct { SManagedObject + + ihosts []cloudprovider.ICloudHost } func NewDatastore(manager *SESXiClient, ds *mo.Datastore, dc *SDatacenter) *SDatastore { @@ -27,7 +43,28 @@ func (self *SDatastore) getDatastore() *mo.Datastore { } func (self *SDatastore) GetGlobalId() string { - return "" + volId, err := self.getVolumeId() + if err != nil { + log.Fatalf("datastore global ID error %s", err) + } + return volId +} + +func (self *SDatastore) GetName() string { + volName, err := self.getVolumeName() + if err != nil { + log.Fatalf("datastore get name error %s", err) + } + return fmt.Sprintf("%s-%s", self.getVolumeType(), volName) +} + +func (self *SDatastore) GetCapacityMB() int { + moStore := self.getDatastore() + return int(moStore.Summary.Capacity / 1024 / 1024) +} + +func (self *SDatastore) GetEnabled() bool { + return true } func (self *SDatastore) GetStatus() string { @@ -46,10 +83,101 @@ func (self *SDatastore) IsEmulated() bool { return false } -func (self *SDatastore) getVolumeId() string { +func (self *SDatastore) getVolumeId() (string, error) { + moStore := self.getDatastore() + vmfsInfo, ok := moStore.Info.(*types.VmfsDatastoreInfo) + if ok { + if vmfsInfo.Vmfs.Local != nil && *vmfsInfo.Vmfs.Local { + host, err := self.getLocalHost() + if err != nil { + return "", err + } + return fmt.Sprintf("%s:%s", host.GetAccessIp(), vmfsInfo.Vmfs.Uuid), nil + } else { + return vmfsInfo.Vmfs.Uuid, nil + } + } + nasInfo, ok := moStore.Info.(*types.NasDatastoreInfo) + if ok { + return fmt.Sprintf("%s:%s", nasInfo.Nas.RemoteHost, nasInfo.Nas.RemotePath), nil + } + if moStore.Summary.Type == "vsan" { + vsanId := moStore.Summary.Url + vsanId = vsanId[strings.Index(vsanId, "vsan:"):] + + endIdx := len(vsanId) + for ; vsanId[endIdx-1] == '/'; endIdx -= 1 { + } + + return vsanId[:endIdx], nil + } + log.Fatalf("unsupported volume type %#v", moStore.Info) + return "", cloudprovider.ErrNotImplemented +} + +func (self *SDatastore) getVolumeType() string { return self.getDatastore().Summary.Type } +func (self *SDatastore) getVolumeName() (string, error) { + moStore := self.getDatastore() + + if self.isLocalVMFS() { + host, err := self.getLocalHost() + if err != nil { + return "", err + } + return fmt.Sprintf("%s-%s", host.GetAccessIp(), moStore.Info.GetDatastoreInfo().Name), nil + } + dc, err := self.GetDatacenter() + if err != nil { + return "", nil + } + return fmt.Sprintf("%s-%s", dc.GetName(), moStore.Info.GetDatastoreInfo().Name), nil +} + +func (self *SDatastore) getAttachedHosts() ([]cloudprovider.ICloudHost, error) { + ihosts := make([]cloudprovider.ICloudHost, 0) + + moStore := self.getDatastore() + for i := 0; i < len(moStore.Host); i += 1 { + idstr := moRefId(moStore.Host[i].Key) + host, err := self.datacenter.GetIHostByMoId(idstr) + if err != nil { + return nil, err + } + ihosts = append(ihosts, host) + } + + return ihosts, nil +} + +func (self *SDatastore) getCachedAttachedHosts() ([]cloudprovider.ICloudHost, error) { + if self.ihosts == nil { + var err error + self.ihosts, err = self.getAttachedHosts() + if err != nil { + return nil, err + } + } + return self.ihosts, nil +} + +func (self *SDatastore) GetAttachedHosts() ([]cloudprovider.ICloudHost, error) { + return self.getCachedAttachedHosts() +} + +func (self *SDatastore) getLocalHost() (cloudprovider.ICloudHost, error) { + hosts, err := self.GetAttachedHosts() + if err != nil { + return nil, err + } + if len(hosts) == 1 { + return hosts[0], nil + } + return nil, cloudprovider.ErrInvalidStatus +} + func (self *SDatastore) GetIStoragecache() cloudprovider.ICloudStoragecache { return nil } @@ -66,31 +194,331 @@ func (self *SDatastore) GetIDisks() ([]cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented } +func (self *SDatastore) isLocalVMFS() bool { + moStore := self.getDatastore() + vmfsInfo, ok := moStore.Info.(*types.VmfsDatastoreInfo) + if ok && vmfsInfo.Vmfs.Local != nil && *vmfsInfo.Vmfs.Local { + return true + } + return false +} + func (self *SDatastore) GetStorageType() string { - return self.getDatastore().Summary.Type + moStore := self.getDatastore() + switch moStore.Summary.Type { + case "VMFS": + if self.isLocalVMFS() { + return models.STORAGE_LOCAL + } else { + return models.STORAGE_NAS + } + case "NFS", "NFS41", "CIFS", "vsan": + return models.STORAGE_NAS + default: + log.Fatalf("unsupported datastore type %s", moStore.Summary.Type) + return "" + } } func (self *SDatastore) GetMediumType() string { - return "" -} - -func (self *SDatastore) GetCapacityMB() int { - return 0 + moStore := self.getDatastore() + vmfsInfo, ok := moStore.Info.(*types.VmfsDatastoreInfo) + if ok && vmfsInfo.Vmfs.Ssd != nil && *vmfsInfo.Vmfs.Ssd { + return models.DISK_TYPE_SSD + } + return models.DISK_TYPE_ROTATE } func (self *SDatastore) GetStorageConf() jsonutils.JSONObject { conf := jsonutils.NewDict() - return conf -} + conf.Add(jsonutils.NewString(self.GetName()), "name") + conf.Add(jsonutils.NewString(self.GetGlobalId()), "id") + conf.Add(jsonutils.NewString(self.GetDatacenterPathString()), "dc_path") + volId, err := self.getVolumeId() + if err != nil { + log.Errorf("getVaolumeId fail %s", err) + } + conf.Add(jsonutils.NewString(volId), "volume_id") -func (self *SDatastore) GetEnabled() bool { - return true + volType := self.getVolumeType() + conf.Add(jsonutils.NewString(volType), "volume_type") + + volName, err := self.getVolumeName() + if err != nil { + log.Errorf("getVaolumeName fail %s", err) + } + conf.Add(jsonutils.NewString(volName), "volume_name") + return conf } func (self *SDatastore) GetManagerId() string { return self.manager.providerId } +func (self *SDatastore) GetUrl() string { + return self.getDatastore().Info.GetDatastoreInfo().Url +} + +func (self *SDatastore) cleanPath(remotePath string) string { + dsName := fmt.Sprintf("[%s]", self.SManagedObject.GetName()) + dsUrl := self.GetUrl() + if strings.HasPrefix(remotePath, dsName) { + remotePath = remotePath[len(dsName):] + } else if strings.HasPrefix(remotePath, dsUrl) { + remotePath = remotePath[len(dsUrl):] + } + return strings.TrimSpace(remotePath) +} + +func pathEscape(path string) string { + segs := strings.Split(path, "/") + for i := 0; i < len(segs); i += 1 { + segs[i] = url.PathEscape(segs[i]) + } + return strings.Join(segs, "/") +} + +func (self *SDatastore) GetPathUrl(remotePath string) string { + remotePath = self.cleanPath(remotePath) + if len(remotePath) == 0 || remotePath[0] != '/' { + remotePath = fmt.Sprintf("/%s", remotePath) + } + httpUrl := fmt.Sprintf("%s/folder%s", self.getManagerUri(), pathEscape(remotePath)) + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(self.SManagedObject.GetName()), "dsName") + params.Add(jsonutils.NewString(self.GetDatacenterPathString()), "dcPath") + + return fmt.Sprintf("%s?%s", httpUrl, params.QueryString()) +} + +func (self *SDatastore) getPathString(path string) string { + for len(path) > 0 && path[0] == '/' { + path = path[1:] + } + return fmt.Sprintf("[%s] %s", self.SManagedObject.GetName(), path) +} + func (self *SDatastore) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented } + +func (self *SDatastore) FileGetContent(ctx context.Context, remotePath string) ([]byte, error) { + url := self.GetPathUrl(remotePath) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + var bytes []byte + + err = self.manager.client.Do(ctx, req, func(resp *http.Response) error { + if resp.StatusCode >= 400 { + return fmt.Errorf("%s", resp.Status) + } + cont, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + bytes = cont + return nil + }) + + return bytes, err +} + +type SDatastoreFileInfo struct { + Url string + Name string + Date time.Time + FileType string + Size uint64 +} + +const ( + fileListPattern = `(?P[^<]+)<\/a>(?P[^<]+)(?P[^<]+)` + fileDateFormat = "02-Jan-2006 15:04" + fileDateFormat2 = "Mon, 2 Jan 2006 15:04:05 GMT" +) + +var ( + fileListRegexp = regexp.MustCompile(fileListPattern) +) + +func (self *SDatastore) ListDir(ctx context.Context, remotePath string) ([]SDatastoreFileInfo, error) { + listContent, err := self.FileGetContent(ctx, remotePath) + if err != nil { + return nil, err + } + ret := make([]SDatastoreFileInfo, 0) + matches := fileListRegexp.FindAllStringSubmatch(string(listContent), -1) + for r := 0; r < len(matches); r += 1 { + url := strings.TrimSpace(matches[r][1]) + name := strings.TrimSpace(matches[r][2]) + dateStr := strings.TrimSpace(matches[r][3]) + sizeStr := strings.TrimSpace(matches[r][4]) + var ftype string + var size uint64 + if sizeStr == "-" { + ftype = "dir" + size = 0 + } else { + ftype = "file" + size, _ = strconv.ParseUint(sizeStr, 10, 64) + } + date, err := time.Parse(fileDateFormat, dateStr) + if err != nil { + return nil, err + } + info := SDatastoreFileInfo{ + Url: url, + Name: name, + FileType: ftype, + Size: size, + Date: date, + } + ret = append(ret, info) + } + + return ret, nil +} + +func (self *SDatastore) CheckFile(ctx context.Context, remotePath string) (*SDatastoreFileInfo, error) { + url := self.GetPathUrl(remotePath) + + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + + var size uint64 + var date time.Time + + err = self.manager.client.Do(ctx, req, func(resp *http.Response) error { + if resp.StatusCode >= 400 { + return fmt.Errorf("%s", resp.Status) + } + sizeStr := resp.Header.Get("Content-Length") + size, _ = strconv.ParseUint(sizeStr, 10, 64) + + dateStr := resp.Header.Get("Date") + date, _ = time.Parse(fileDateFormat2, dateStr) + return nil + }) + + if err != nil { + return nil, err + } + return &SDatastoreFileInfo{Date: date, Size: size}, nil +} + +func (self *SDatastore) Download(ctx context.Context, remotePath string, writer io.Writer) error { + url := self.GetPathUrl(remotePath) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + err = self.manager.client.Do(ctx, req, func(resp *http.Response) error { + if resp.StatusCode >= 400 { + return fmt.Errorf("%s", resp.Status) + } + buffer := make([]byte, 4096) + for { + rn, re := resp.Body.Read(buffer) + if rn > 0 { + wo := 0 + for wo < rn { + wn, we := writer.Write(buffer[wo:rn]) + if we != nil { + return we + } + wo += wn + } + } + if re != nil { + if re != io.EOF { + return re + } else { + break + } + } + } + return nil + }) + + return err +} + +func (self *SDatastore) Upload(ctx context.Context, remotePath string, body io.Reader) error { + url := self.GetPathUrl(remotePath) + + req, err := http.NewRequest("PUT", url, body) + if err != nil { + return err + } + + err = self.manager.client.Do(ctx, req, func(resp *http.Response) error { + if resp.StatusCode >= 400 { + return fmt.Errorf("%s", resp.Status) + } + _, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + // log.Debugf("upload respose %s", buffer) + return nil + }) + + return err +} + +func (self *SDatastore) FilePutContent(ctx context.Context, remotePath string, content string) error { + return self.Upload(ctx, remotePath, strings.NewReader(content)) +} + +func (self *SDatastore) Delete(ctx context.Context, remotePath string) error { + url := self.GetPathUrl(remotePath) + + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + err = self.manager.client.Do(ctx, req, func(resp *http.Response) error { + if resp.StatusCode >= 400 { + return fmt.Errorf("%s", resp.Status) + } + _, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + // log.Debugf("delete respose %s", buffer) + return nil + }) + + return err +} + +func (self *SDatastore) DeleteVmdk(ctx context.Context, remotePath string) error { + vmdkContent, err := self.FileGetContent(ctx, remotePath) + if err != nil { + return err + } + vmdkInfo, err := vmdkutils.Parse(string(vmdkContent)) + if err != nil { + return err + } + err = self.Delete(ctx, remotePath) + if err != nil { + return err + } + if len(vmdkInfo.ExtentFile) > 0 { + err = self.Delete(ctx, path.Join(remotePath, vmdkInfo.ExtentFile)) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/util/esxi/utils.go b/pkg/util/esxi/utils.go new file mode 100644 index 0000000000..1cb913dd04 --- /dev/null +++ b/pkg/util/esxi/utils.go @@ -0,0 +1,83 @@ +package esxi + +import ( + "context" + "reflect" + + "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vim25" + "github.com/vmware/govmomi/vim25/types" + + "yunion.io/x/log" +) + +func Properties(c *vim25.Client, ctx context.Context, r types.ManagedObjectReference, ps []string, dst interface{}) error { + return property.DefaultCollector(c).RetrieveOne(ctx, r, ps, dst) +} + +func moRefId(ref types.ManagedObjectReference) string { + return ref.Value +} + +func moRefType(ref types.ManagedObjectReference) string { + return ref.Type +} + +func FetchAnonymousFieldValue(val interface{}, target interface{}) bool { + return fetchAnonymousFieldValue(reflect.Indirect(reflect.ValueOf(val)), + reflect.Indirect(reflect.ValueOf(target))) +} + +func fetchAnonymousFieldValue(value reflect.Value, target reflect.Value) bool { + for i := 0; i < value.NumField(); i += 1 { + fieldValue := value.Field(i) + fieldStruct := value.Type().Field(i) + if fieldStruct.Anonymous && fieldStruct.Type.Kind() == reflect.Struct { + if fieldStruct.Type == target.Type() { + target.Set(fieldValue) + return true + } + succ := fetchAnonymousFieldValue(fieldValue, target) + if succ { + return true + } + } + } + return false +} + +func StructContains(type1 reflect.Type, type2 reflect.Type) bool { + if type1.Kind() != reflect.Struct || type2.Kind() != reflect.Struct { + log.Errorf("types should be struct!") + return false + } + if type1 == type2 { + return true + } + for i := 0; i < type1.NumField(); i += 1 { + field := type1.Field(i) + if field.Anonymous && field.Type.Kind() == reflect.Struct { + contains := StructContains(field.Type, type2) + if contains { + return true + } + } + } + return false +} + +func reverseArray(array interface{}) { + arrayValue := reflect.Indirect(reflect.ValueOf(array)) + if arrayValue.Kind() != reflect.Slice && arrayValue.Kind() != reflect.Array { + log.Errorf("reverse non array or slice") + return + } + tmp := reflect.Indirect(reflect.New(arrayValue.Type().Elem())) + for i, j := 0, arrayValue.Len()-1; i < j; i, j = i+1, j-1 { + tmpi := arrayValue.Index(i) + tmpj := arrayValue.Index(j) + tmp.Set(tmpi) + tmpi.Set(tmpj) + tmpj.Set(tmp) + } +} diff --git a/pkg/util/esxi/vdisk.go b/pkg/util/esxi/vdisk.go new file mode 100644 index 0000000000..8334b7d22b --- /dev/null +++ b/pkg/util/esxi/vdisk.go @@ -0,0 +1,211 @@ +package esxi + +import ( + "fmt" + "path" + + "github.com/vmware/govmomi/vim25/types" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "context" + "strings" + "time" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" +) + +type SVirtualDisk struct { + SVirtualDevice +} + +func NewVirtualDisk(vm *SVirtualMachine, dev types.BaseVirtualDevice, index int) SVirtualDisk { + return SVirtualDisk{ + NewVirtualDevice(vm, dev, index), + } +} + +func (disk *SVirtualDisk) getVirtualDisk() *types.VirtualDisk { + return disk.dev.(*types.VirtualDisk) +} + +func (disk *SVirtualDisk) getBackingInfo() *types.VirtualDiskFlatVer2BackingInfo { + backing := disk.getVirtualDisk().Backing + switch backing.(type) { + case *types.VirtualDiskFlatVer2BackingInfo: + return backing.(*types.VirtualDiskFlatVer2BackingInfo) + case *types.VirtualDeviceFileBackingInfo: + case *types.VirtualDiskFlatVer1BackingInfo: + case *types.VirtualDiskLocalPMemBackingInfo: + case *types.VirtualDiskRawDiskMappingVer1BackingInfo: + case *types.VirtualDiskSeSparseBackingInfo: + case *types.VirtualDiskSparseVer1BackingInfo: + case *types.VirtualDiskSparseVer2BackingInfo: + case *types.VirtualFloppyImageBackingInfo: + case *types.VirtualNVDIMMBackingInfo: + case *types.VirtualParallelPortFileBackingInfo: + case *types.VirtualSerialPortFileBackingInfo: + case *types.VirtualCdromIsoBackingInfo: + } + log.Fatalf("unsupported backing info %T", backing) + return nil +} + +func (disk *SVirtualDisk) GetId() string { + backing := disk.getBackingInfo() + return backing.Uuid +} + +func (disk *SVirtualDisk) GetName() string { + backing := disk.getBackingInfo() + return path.Base(backing.FileName) +} + +func (disk *SVirtualDisk) GetGlobalId() string { + return fmt.Sprintf("%s-%s", disk.vm.GetGlobalId(), disk.GetId()) +} + +func (disk *SVirtualDisk) GetStatus() string { + return models.DISK_READY +} + +func (disk *SVirtualDisk) Refresh() error { + return nil +} + +func (disk *SVirtualDisk) IsEmulated() bool { + return false +} + +func (disk *SVirtualDisk) GetMetadata() *jsonutils.JSONDict { + return nil +} + +func (disk *SVirtualDisk) GetDiskSizeMB() int { + capa := disk.getVirtualDisk().CapacityInBytes + if capa == 0 { + capa = disk.getVirtualDisk().CapacityInKB * 1024 + } + return int(capa / 1024 / 1024) +} + +func (disk *SVirtualDisk) getFilename() string { + return disk.getBackingInfo().FileName +} + +func (disk *SVirtualDisk) GetDiskFormat() string { + return "vmdk" +} + +func (disk *SVirtualDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + dsObj := disk.getBackingInfo().Datastore + dc, err := disk.vm.GetDatacenter() + if err != nil { + log.Errorf("fail to find datacenter %s", err) + return nil, err + } + istorage, err := dc.GetIStorageByMoId(moRefId(*dsObj)) + if err != nil { + return nil, err + } + return istorage, nil +} + +func (disk *SVirtualDisk) GetIsAutoDelete() bool { + return true +} + +func (disk *SVirtualDisk) GetTemplateId() string { + backing := disk.getBackingInfo() + if backing.Parent != nil { + return path.Base(backing.Parent.FileName) + } + return "" +} + +func (disk *SVirtualDisk) GetDiskType() string { + backing := disk.getBackingInfo() + if backing.Parent != nil { + // log.Infof("%s", backing.Parent.FileName) + return models.DISK_TYPE_SYS + } + return models.DISK_TYPE_DATA +} + +func (disk *SVirtualDisk) GetFsFormat() string { + return "" +} + +func (disk *SVirtualDisk) getDiskMode() string { + backing := disk.getBackingInfo() + return backing.DiskMode +} + +func (disk *SVirtualDisk) GetIsNonPersistent() bool { + return disk.getDiskMode() == "persistent" +} + +func (disk *SVirtualDisk) GetDriver() string { + controller := disk.vm.getVdev(disk.getControllerKey()) + name := controller.GetDriver() + name = strings.Replace(name, "controller", "", -1) + mapping := map[string]string{ + "ahci": "sata", + "parascsi": "pvscsi", + "buslogic": "scsi", + "lsilogic": "scsi", + "lsilogicsas": "scsi", + } + return mapping[name] +} + +func (disk *SVirtualDisk) GetCacheMode() string { + backing := disk.getBackingInfo() + if backing.WriteThrough != nil && *backing.WriteThrough { + return "writethrough" + } else { + return "none" + } +} + +func (disk *SVirtualDisk) GetMountpoint() string { + return "" +} + +func (disk *SVirtualDisk) Delete(ctx context.Context) error { + istorage, err := disk.GetIStorage() + if err != nil { + return err + } + ds := istorage.(*SDatastore) + return ds.DeleteVmdk(ctx, disk.getFilename()) +} + +func (disk *SVirtualDisk) CreateISnapshot(ctx context.Context, name string, desc string) (cloudprovider.ICloudSnapshot, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (disk *SVirtualDisk) GetISnapshot(idStr string) (cloudprovider.ICloudSnapshot, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (disk *SVirtualDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (disk *SVirtualDisk) Resize(ctx context.Context, newSize int64) error { + return cloudprovider.ErrNotImplemented +} + +func (disk *SVirtualDisk) Reset(ctx context.Context, snapshotId string) error { + return cloudprovider.ErrNotImplemented +} + +func (disk *SVirtualDisk) GetBillingType() string { + return "" +} + +func (disk *SVirtualDisk) GetExpiredAt() time.Time { + return time.Time{} +} diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 623a4da458..2000bc0890 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -1,6 +1,7 @@ package esxi import ( + "context" "fmt" "time" @@ -11,20 +12,34 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/secrules" + "reflect" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/pkg/util/netutils" + "yunion.io/yunioncloud/pkg/util/regutils" ) -var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "runtime", "summary"} +var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "runtime", "summary", "config", "guest"} type SVirtualMachine struct { SManagedObject + vnics []SVirtualNIC + vdisks []SVirtualDisk + vga SVirtualVGA + cdroms []SVirtualCdrom + devs map[int32]SVirtualDevice + + guestIps map[string]string + host *SHost } func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacenter, host *SHost) *SVirtualMachine { - return &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc), host: host} + svm := &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc), host: host} + svm.fetchHardwareInfo() + return svm } func (self *SVirtualMachine) GetMetadata() *jsonutils.JSONDict { @@ -62,30 +77,30 @@ func (self *SVirtualMachine) GetStatus() string { } func (self *SVirtualMachine) Refresh() error { - return cloudprovider.ErrNotImplemented + return nil } func (self *SVirtualMachine) IsEmulated() bool { return false } -func (self *SVirtualMachine) DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error { +func (self *SVirtualMachine) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { return cloudprovider.ErrNotImplemented } -func (self *SVirtualMachine) RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SVirtualMachine) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { return "", cloudprovider.ErrNotImplemented } -func (self *SVirtualMachine) UpdateVM(name string) error { +func (self *SVirtualMachine) UpdateVM(ctx context.Context, name string) error { return cloudprovider.ErrNotImplemented } -func (self *SVirtualMachine) DetachDisk(diskId string) error { +func (self *SVirtualMachine) DetachDisk(ctx context.Context, diskId string) error { return cloudprovider.ErrNotImplemented } -func (self *SVirtualMachine) AttachDisk(diskId string) error { +func (self *SVirtualMachine) AttachDisk(ctx context.Context, diskId string) error { return cloudprovider.ErrNotImplemented } @@ -94,19 +109,45 @@ func (self *SVirtualMachine) getUuid() string { } func (self *SVirtualMachine) GetCreateTime() time.Time { - return time.Time{} + moVM := self.getVirtualMachine() + ctm := moVM.Config.CreateDate + if ctm != nil { + return *ctm + } else { + return time.Time{} + } } func (self *SVirtualMachine) GetIHost() cloudprovider.ICloudHost { - return self.host + // moVM := self.getVirtualMachine() + // log.Debugf("%#v", moVM.Parent) + me := self.findInParents("HostSystem") + if me == nil { + log.Errorf("fail to find vm host??? %s", self.GetName()) + return self.host + } + ihost, err := self.manager.FindHostByMoId(me.Self.Value) + if err != nil { + log.Errorf("fail to find host %s for vm %s???", me.Self.Value, self.GetName()) + return nil + } + return ihost } func (self *SVirtualMachine) GetIDisks() ([]cloudprovider.ICloudDisk, error) { - return nil, cloudprovider.ErrNotImplemented + idisks := make([]cloudprovider.ICloudDisk, len(self.vdisks)) + for i := 0; i < len(self.vdisks); i += 1 { + idisks[i] = &(self.vdisks[i]) + } + return idisks, nil } func (self *SVirtualMachine) GetINics() ([]cloudprovider.ICloudNic, error) { - return nil, cloudprovider.ErrNotImplemented + inics := make([]cloudprovider.ICloudNic, len(self.vnics)) + for i := 0; i < len(self.vnics); i += 1 { + inics[i] = &(self.vnics[i]) + } + return inics, nil } func (self *SVirtualMachine) GetIEIP() (cloudprovider.ICloudEIP, error) { @@ -114,12 +155,10 @@ func (self *SVirtualMachine) GetIEIP() (cloudprovider.ICloudEIP, error) { } func (self *SVirtualMachine) GetVcpuCount() int8 { - // ret = self.obj.summary.config.numCpu return int8(self.getVirtualMachine().Summary.Config.NumCpu) } func (self *SVirtualMachine) GetVmemSizeMB() int { - // self.obj.summary.config.memorySizeMB return int(self.getVirtualMachine().Summary.Config.MemorySizeMB) } @@ -135,6 +174,36 @@ func (self *SVirtualMachine) GetVdi() string { return "vmrc" } +func (self *SVirtualMachine) GetGuestFamily() string { + moVM := self.getVirtualMachine() + return moVM.Config.AlternateGuestName +} + +func (self *SVirtualMachine) GetGuestId() string { + moVM := self.getVirtualMachine() + return moVM.Config.GuestId +} + +func (self *SVirtualMachine) GetGuestFullName() string { + moVM := self.getVirtualMachine() + return moVM.Config.GuestFullName +} + +func (self *SVirtualMachine) GetGuestState() string { + moVM := self.getVirtualMachine() + return moVM.Guest.GuestState +} + +func (self *SVirtualMachine) GetGuestToolsStatus() string { + moVM := self.getVirtualMachine() + return string(moVM.Guest.ToolsStatus) +} + +func (self *SVirtualMachine) GetGuestToolsRunningStatus() string { + moVM := self.getVirtualMachine() + return string(moVM.Guest.ToolsRunningStatus) +} + func (self *SVirtualMachine) GetOSType() string { return "" } @@ -163,18 +232,97 @@ func (self *SVirtualMachine) GetHypervisor() string { return models.HYPERVISOR_ESXI } -// GetSecurityGroup() ICloudSecurityGroup - -func (self *SVirtualMachine) StartVM() error { - return cloudprovider.ErrNotImplemented +func (self *SVirtualMachine) getVmObj() *object.VirtualMachine { + return object.NewVirtualMachine(self.manager.client.Client, self.getVirtualMachine().Self) } -func (self *SVirtualMachine) StopVM(isForce bool) error { - return cloudprovider.ErrNotImplemented +func (self *SVirtualMachine) StartVM(ctx context.Context) error { + err := self.makeNicsStartConnected(ctx) + if err != nil { + return err + } + task, err := self.getVmObj().PowerOn(ctx) + if err != nil { + return err + } + return task.Wait(ctx) } -func (self *SVirtualMachine) DeleteVM() error { - return cloudprovider.ErrNotImplemented +func (self *SVirtualMachine) makeNicsStartConnected(ctx context.Context) error { + spec := types.VirtualMachineConfigSpec{} + spec.DeviceChange = make([]types.BaseVirtualDeviceConfigSpec, len(self.vnics)) + for i := 0; i < len(self.vnics); i += 1 { + spec.DeviceChange[i] = makeNicStartConnected(&self.vnics[i]) + } + task, err := self.getVmObj().Reconfigure(ctx, spec) + if err != nil { + return err + } + return task.Wait(ctx) +} + +func makeNicStartConnected(nic *SVirtualNIC) *types.VirtualDeviceConfigSpec { + editSpec := types.VirtualDeviceConfigSpec{} + editSpec.Operation = types.VirtualDeviceConfigSpecOperationEdit + editSpec.FileOperation = "" + editSpec.Device = nic.dev + editSpec.Device.GetVirtualDevice().Connectable.StartConnected = true + return &editSpec +} + +func (self *SVirtualMachine) StopVM(ctx context.Context, isForce bool) error { + task, err := self.getVmObj().PowerOff(ctx) + if err != nil { + return err + } + return task.Wait(ctx) +} + +func (self *SVirtualMachine) doDelete(ctx context.Context) error { + task, err := self.getVmObj().Destroy(ctx) + if err != nil { + return err + } + return task.Wait(ctx) +} + +func (self *SVirtualMachine) DeleteVM(ctx context.Context) error { + for i := 0; i < len(self.vdisks); i += 1 { + err := self.doDetachAndDeleteDisk(ctx, &self.vdisks[i]) + if err != nil { + return err + } + } + return self.doDelete(ctx) +} + +func (self *SVirtualMachine) doDetachAndDeleteDisk(ctx context.Context, vdisk *SVirtualDisk) error { + return self.doDetachDisk(ctx, vdisk, true) +} + +func (self *SVirtualMachine) doDetachDisk(ctx context.Context, vdisk *SVirtualDisk, remove bool) error { + removeSpec := types.VirtualDeviceConfigSpec{} + removeSpec.Operation = types.VirtualDeviceConfigSpecOperationRemove + removeSpec.Device = vdisk.dev + + spec := types.VirtualMachineConfigSpec{} + spec.DeviceChange = []types.BaseVirtualDeviceConfigSpec{&removeSpec} + + task, err := self.getVmObj().Reconfigure(ctx, spec) + if err != nil { + return err + } + + err = task.Wait(ctx) + if err != nil { + return err + } + + if !remove { + return nil + } + + return vdisk.Delete(ctx) } func (self *SVirtualMachine) GetVNCInfo() (jsonutils.JSONObject, error) { @@ -234,7 +382,7 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { return ret, nil } -func (dc *SVirtualMachine) ChangeConfig(instanceId string, ncpu int, vmem int) error { +func (dc *SVirtualMachine) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { return cloudprovider.ErrNotImplemented } @@ -249,3 +397,72 @@ func (self *SVirtualMachine) GetExpiredAt() time.Time { func (self *SVirtualMachine) UpdateUserData(userData string) error { return nil } + +func (self *SVirtualMachine) fetchHardwareInfo() { + self.vnics = make([]SVirtualNIC, 0) + self.vdisks = make([]SVirtualDisk, 0) + self.cdroms = make([]SVirtualCdrom, 0) + self.devs = make(map[int32]SVirtualDevice) + + moVM := self.getVirtualMachine() + + for i := 0; i < len(moVM.Config.Hardware.Device); i += 1 { + dev := moVM.Config.Hardware.Device[i] + devType := reflect.Indirect(reflect.ValueOf(dev)).Type() + + etherType := reflect.TypeOf((*types.VirtualEthernetCard)(nil)).Elem() + diskType := reflect.TypeOf((*types.VirtualDisk)(nil)).Elem() + vgaType := reflect.TypeOf((*types.VirtualMachineVideoCard)(nil)).Elem() + cdromType := reflect.TypeOf((*types.VirtualCdrom)(nil)).Elem() + + if StructContains(devType, etherType) { + self.vnics = append(self.vnics, NewVirtualNIC(self, dev, len(self.vnics))) + } else if StructContains(devType, diskType) { + self.vdisks = append(self.vdisks, NewVirtualDisk(self, dev, len(self.vnics))) + } else if StructContains(devType, vgaType) { + self.vga = NewVirtualVGA(self, dev, 0) + } else if StructContains(devType, cdromType) { + self.cdroms = append(self.cdroms, NewVirtualCdrom(self, dev, len(self.cdroms))) + } + vdev := NewVirtualDevice(self, dev, 0) + self.devs[vdev.getKey()] = vdev + } +} + +func (self *SVirtualMachine) getVdev(key int32) SVirtualDevice { + return self.devs[key] +} + +func (self *SVirtualMachine) fetchGuestIps() map[string]string { + guestIps := make(map[string]string) + moVM := self.getVirtualMachine() + for _, net := range moVM.Guest.Net { + mac := netutils.FormatMacAddr(net.MacAddress) + for _, ip := range net.IpAddress { + if regutils.MatchIP4Addr(ip) { + guestIps[mac] = ip + break + } + } + } + return guestIps +} + +func (self *SVirtualMachine) getGuestIps() map[string]string { + if self.guestIps == nil { + self.guestIps = self.fetchGuestIps() + } + return self.guestIps +} + +func (self *SVirtualMachine) GetIps() []string { + ips := make([]string, 0) + for _, ip := range self.getGuestIps() { + ips = append(ips, ip) + } + return ips +} + +func (self *SVirtualMachine) GetVGADevice() string { + return fmt.Sprintf("%s", self.vga.String()) +} diff --git a/pkg/util/esxi/vnic.go b/pkg/util/esxi/vnic.go new file mode 100644 index 0000000000..ae8196ea95 --- /dev/null +++ b/pkg/util/esxi/vnic.go @@ -0,0 +1,44 @@ +package esxi + +import ( + "github.com/vmware/govmomi/vim25/types" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/pkg/util/netutils" +) + +type SVirtualNIC struct { + SVirtualDevice +} + +func NewVirtualNIC(vm *SVirtualMachine, dev types.BaseVirtualDevice, index int) SVirtualNIC { + return SVirtualNIC{ + NewVirtualDevice(vm, dev, index), + } +} + +func (nic *SVirtualNIC) getVirtualEthernetCard() *types.VirtualEthernetCard { + card := types.VirtualEthernetCard{} + if FetchAnonymousFieldValue(nic.dev, &card) { + return &card + } + return nil +} + +func (nic *SVirtualNIC) GetIP() string { + guestIps := nic.vm.getGuestIps() + if ip, ok := guestIps[nic.GetMAC()]; ok { + return ip + } + log.Warningf("cannot find ip for mac %s", nic.GetMAC()) + return "" +} + +func (nic *SVirtualNIC) GetMAC() string { + return netutils.FormatMacAddr(nic.getVirtualEthernetCard().MacAddress) +} + +func (nic *SVirtualNIC) GetINetwork() cloudprovider.ICloudNetwork { + return nil +} diff --git a/pkg/util/printutils/printgetter.go b/pkg/util/printutils/printgetter.go index 1cca6b48c4..1765efd1ee 100644 --- a/pkg/util/printutils/printgetter.go +++ b/pkg/util/printutils/printgetter.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/utils" ) @@ -24,11 +25,18 @@ func getter2json(obj interface{}) jsonutils.JSONObject { methodName := method.Name methodType := methodValue.Type() - if strings.HasPrefix(methodName, "Get") && methodType.NumIn() == 0 && methodType.NumOut() == 1 { + if strings.HasPrefix(methodName, "Get") && methodType.NumIn() == 0 && methodType.NumOut() >= 1 { fieldName := utils.CamelSplit(methodName[3:], "_") out := methodValue.Call([]reflect.Value{}) - if len(out) == 1 { + if len(out) == 1 && !gotypes.IsNil(out[0].Interface()) { jsonDict.Add(jsonutils.Marshal(out[0].Interface()), fieldName) + } else if len(out) == 2 { + err, ok := out[1].Interface().(error) + if ok { + if err != nil && !gotypes.IsNil(out[0].Interface()) { + jsonDict.Add(jsonutils.Marshal(out[0].Interface()), fieldName) + } + } } } } diff --git a/pkg/util/qcloud/disk.go b/pkg/util/qcloud/disk.go index 61377bc936..8705edcdfe 100644 --- a/pkg/util/qcloud/disk.go +++ b/pkg/util/qcloud/disk.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -150,11 +151,11 @@ func (self *SRegion) DeleteDisk(diskId string) error { return err } -func (self *SDisk) Delete() error { +func (self *SDisk) Delete(ctx context.Context) error { return self.storage.zone.region.DeleteDisk(self.DiskId) } -func (self *SRegion) ResizeDisk(diskId string, sizeGb int64) error { +func (self *SRegion) ResizeDisk(ctx context.Context, diskId string, sizeGb int64) error { params := make(map[string]string) params["DiskId"] = diskId params["DiskSize"] = fmt.Sprintf("%d", sizeGb) @@ -175,8 +176,8 @@ func (self *SRegion) ResizeDisk(diskId string, sizeGb int64) error { } } -func (self *SDisk) Resize(size int64) error { - return self.storage.zone.region.ResizeDisk(self.DiskId, size) +func (self *SDisk) Resize(ctx context.Context, size int64) error { + return self.storage.zone.region.ResizeDisk(ctx, self.DiskId, size) } func (self *SDisk) GetName() string { @@ -194,8 +195,8 @@ func (self *SDisk) IsEmulated() bool { return false } -func (self *SDisk) GetIStorge() cloudprovider.ICloudStorage { - return self.storage +func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return self.storage, nil } func (self *SDisk) GetStatus() string { @@ -215,7 +216,7 @@ func (self *SDisk) Refresh() error { return jsonutils.Update(self, new) } -func (self *SDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) { +func (self *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { snapshotId, err := self.storage.zone.region.CreateSnapshot(self.DiskId, name, desc) if err != nil { log.Errorf("createSnapshot fail %s", err) @@ -344,7 +345,7 @@ func (self *SRegion) ResetDisk(diskId, snapshotId string) error { return nil } -func (self *SDisk) Reset(snapshotId string) error { +func (self *SDisk) Reset(ctx context.Context, snapshotId string) error { return self.storage.zone.region.ResetDisk(self.DiskId, snapshotId) } diff --git a/pkg/util/qcloud/host.go b/pkg/util/qcloud/host.go index 67f362deb4..53ed257cb1 100644 --- a/pkg/util/qcloud/host.go +++ b/pkg/util/qcloud/host.go @@ -270,3 +270,15 @@ func (self *SHost) GetSysInfo() jsonutils.JSONObject { func (self *SHost) IsEmulated() bool { return true } + +func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (host *SHost) GetIsMaintenance() bool { + return false +} + +func (host *SHost) GetVersion() string { + return QCLOUD_API_VERSION +} diff --git a/pkg/util/qcloud/instance.go b/pkg/util/qcloud/instance.go index aa51d86820..ba0103bffa 100644 --- a/pkg/util/qcloud/instance.go +++ b/pkg/util/qcloud/instance.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -316,7 +317,7 @@ func (self *SInstance) GetHypervisor() string { return models.HYPERVISOR_QCLOUD } -func (self *SInstance) StartVM() error { +func (self *SInstance) StartVM(ctx context.Context) error { timeout := 300 * time.Second interval := 15 * time.Second @@ -341,7 +342,7 @@ func (self *SInstance) StartVM() error { return cloudprovider.ErrTimeout } -func (self *SInstance) StopVM(isForce bool) error { +func (self *SInstance) StopVM(ctx context.Context, isForce bool) error { err := self.host.zone.region.StopVM(self.InstanceId, isForce) if err != nil { return err @@ -361,11 +362,11 @@ func (self *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { return ret, nil } -func (self *SInstance) UpdateVM(name string) error { +func (self *SInstance) UpdateVM(ctx context.Context, name string) error { return self.host.zone.region.UpdateVM(self.InstanceId, name) } -func (self *SInstance) DeployVM(name string, password string, publicKey string, deleteKeypair bool, description string) error { +func (self *SInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { var keypairName string if len(publicKey) > 0 { var err error @@ -378,7 +379,7 @@ func (self *SInstance) DeployVM(name string, password string, publicKey string, return self.host.zone.region.DeployVM(self.InstanceId, name, password, keypairName, deleteKeypair, description) } -func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { +func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) { keypair := "" if len(publicKey) > 0 { var err error @@ -391,7 +392,7 @@ func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey stri if err != nil { return "", err } - self.StopVM(true) + self.StopVM(ctx, true) instance, err := self.host.zone.region.GetInstance(self.InstanceId) if err != nil { return "", err @@ -399,15 +400,15 @@ func (self *SInstance) RebuildRoot(imageId string, passwd string, publicKey stri return instance.SystemDisk.DiskId, nil } -func (self *SInstance) ChangeConfig(instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { return self.host.zone.region.ChangeVMConfig(self.Placement.Zone, self.InstanceId, ncpu, vmem, nil) } -func (self *SInstance) AttachDisk(diskId string) error { +func (self *SInstance) AttachDisk(ctx context.Context, diskId string) error { return self.host.zone.region.AttachDisk(self.InstanceId, diskId) } -func (self *SInstance) DetachDisk(diskId string) error { +func (self *SInstance) DetachDisk(ctx context.Context, diskId string) error { return self.host.zone.region.DetachDisk(self.InstanceId, diskId) } @@ -528,7 +529,7 @@ func (self *SRegion) StopVM(instanceId string, isForce bool) error { return self.doStopVM(instanceId, isForce) } -func (self *SRegion) DeleteVM(instanceId string) error { +func (self *SRegion) DeleteVM(ctx context.Context, instanceId string) error { status, err := self.GetInstanceStatus(instanceId) if err != nil { log.Errorf("Fail to get instance status on DeleteVM: %s", err) @@ -582,9 +583,9 @@ func (self *SRegion) DeployVM(instanceId string, name string, password string, k return nil } -func (self *SInstance) DeleteVM() error { +func (self *SInstance) DeleteVM(ctx context.Context) error { for { - err := self.host.zone.region.DeleteVM(self.InstanceId) + err := self.host.zone.region.DeleteVM(ctx, self.InstanceId) if err != nil { // if isError(err, "IncorrectInstanceStatus.Initializing") { // log.Infof("The instance is initializing, try later ...") diff --git a/pkg/util/qcloud/localdisk.go b/pkg/util/qcloud/localdisk.go index ad91c8a764..ab100b7a69 100644 --- a/pkg/util/qcloud/localdisk.go +++ b/pkg/util/qcloud/localdisk.go @@ -3,6 +3,7 @@ package qcloud import ( "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -20,11 +21,11 @@ func (self *SLocalDisk) GetMetadata() *jsonutils.JSONDict { return nil } -func (self *SLocalDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) { +func (self *SLocalDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { return nil, cloudprovider.ErrNotSupported } -func (self *SLocalDisk) Delete() error { +func (self *SLocalDisk) Delete(ctx context.Context) error { return nil } @@ -83,7 +84,7 @@ func (self *SLocalDisk) Refresh() error { return nil } -func (self *SLocalDisk) Reset(snapshotId string) error { +func (self *SLocalDisk) Reset(ctx context.Context, snapshotId string) error { return cloudprovider.ErrNotSupported } @@ -119,10 +120,10 @@ func (self *SLocalDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) return nil, nil } -func (self *SLocalDisk) GetIStorge() cloudprovider.ICloudStorage { - return self.storage +func (self *SLocalDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return self.storage, nil } -func (self *SLocalDisk) Resize(size int64) error { +func (self *SLocalDisk) Resize(ctx context.Context, size int64) error { return cloudprovider.ErrNotSupported } diff --git a/pkg/util/qcloud/provider/provider.go b/pkg/util/qcloud/provider/provider.go index 206fb909bf..9f34d9123a 100644 --- a/pkg/util/qcloud/provider/provider.go +++ b/pkg/util/qcloud/provider/provider.go @@ -35,6 +35,10 @@ func (self *SQcloudProvider) IsPublicCloud() bool { return true } +func (self *SQcloudProvider) IsOnPremiseInfrastructure() bool { + return false +} + func (self *SQcloudProvider) GetId() string { return qcloud.CLOUD_PROVIDER_QCLOUD } @@ -82,3 +86,7 @@ func (self *SQcloudProvider) GetIStoragecacheById(id string) (cloudprovider.IClo func (self *SQcloudProvider) GetBalance() (float64, error) { return 0.0, nil } + +func (self *SQcloudProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { + return nil, cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/vmdkutils/vmdkutils.go b/pkg/util/vmdkutils/vmdkutils.go new file mode 100644 index 0000000000..b08b45ecf9 --- /dev/null +++ b/pkg/util/vmdkutils/vmdkutils.go @@ -0,0 +1,79 @@ +package vmdkutils + +import ( + "bufio" + "io" + "regexp" + "strings" + + "fmt" + "strconv" + "yunion.io/x/pkg/utils" +) + +type SVMDKInfo struct { + ExtentFile string + Heads int64 + Sectors int64 + Cylinders int64 + CID string + LongCID string + UUID string + AdapterType string + VirtualHWVersion string +} + +const ( + extentPatternString = `^RW \d+ VMFS\w* \"(?P[^"]+)` +) + +var ( + extentPatternRegexp = regexp.MustCompile(extentPatternString) +) + +func Parse(content string) (*SVMDKInfo, error) { + return ParseStream(strings.NewReader(content)) +} + +func ParseStream(stream io.Reader) (*SVMDKInfo, error) { + info := SVMDKInfo{} + scanner := bufio.NewScanner(stream) + findExtent := false + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + matches := extentPatternRegexp.FindStringSubmatch(line) + if len(matches) > 0 { + // log.Debugf("%#v", matches) + info.ExtentFile = matches[1] + findExtent = true + } else { + equalPos := strings.IndexByte(line, '=') + if equalPos > 0 { + key := strings.TrimSpace(line[:equalPos]) + value := utils.Unquote(strings.TrimSpace(line[equalPos+1:])) + switch key { + case "CID": + info.CID = value + case "ddb.uuid": + info.UUID = value + case "ddb.geometry.cylinders": + info.Cylinders, _ = strconv.ParseInt(value, 10, 64) + case "ddb.geometry.heads": + info.Heads, _ = strconv.ParseInt(value, 10, 64) + case "ddb.geometry.sectors": + info.Sectors, _ = strconv.ParseInt(value, 10, 64) + case "ddb.longContentID": + info.LongCID = value + case "ddb.adapterType": + info.AdapterType = value + case "ddb.virtualHWVersion": + info.VirtualHWVersion = value + } + } + } + } + if !findExtent { + return nil, fmt.Errorf("not a vmdk file") + } + return &info, nil +} diff --git a/pkg/util/vmdkutils/vmdkutils_test.go b/pkg/util/vmdkutils/vmdkutils_test.go new file mode 100644 index 0000000000..8fee91aeef --- /dev/null +++ b/pkg/util/vmdkutils/vmdkutils_test.go @@ -0,0 +1,44 @@ +package vmdkutils + +import "testing" + +const ( + VMDKContent = ` version=1 + encoding="UTF-8" + CID=ab798ecb + parentCID=ffffffff + isNativeSnapshot="no" + createType="vmfs" + + # Extent description + RW 62914560 VMFS "c8336ec2-3885-4205-a2f1-095e48228a56-flat.vmdk" + + # The Disk Data Base + #DDB + + ddb.adapterType = "lsilogic" + ddb.deletable = "false" + ddb.geometry.cylinders = "62415" + ddb.geometry.heads = "16" + ddb.geometry.sectors = "63" + ddb.longContentID = "4d1762d22407e6b7e3e63557ab798ecb" + ddb.thinProvisioned = "1" + ddb.uuid = "60 00 C2 97 32 7d ac 2b-ac 97 a4 95 5b a6 0b c0" + ddb.virtualHWVersion = "13" +` +) + +func TestParseStream(t *testing.T) { + info, err := Parse(VMDKContent) + if err != nil { + t.Errorf("parse error %s", err) + } + t.Logf("%#v", info) + + _, err = Parse("") + if err != nil { + t.Logf("parse error %s", err) + } else { + t.Errorf("should parse error") + } +} From 94c49ded2e5dad1a91f88928fac9b818f98d0971 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sun, 25 Nov 2018 08:21:24 +0800 Subject: [PATCH 02/10] esxi support --- cmd/climc/shell/cloudmeta.go | 4 +- pkg/cloudcommon/db/taskman/interface.go | 3 + pkg/cloudcommon/db/taskman/localtaskworker.go | 2 +- pkg/cloudcommon/db/taskman/tasks.go | 10 +- pkg/cloudprovider/cloudprovider.go | 10 +- pkg/cloudprovider/fakeregion.go | 76 +++++ pkg/cloudprovider/resources.go | 38 ++- pkg/compute/guestdrivers/aliyun.go | 213 ------------- pkg/compute/guestdrivers/aws.go | 77 +---- pkg/compute/guestdrivers/azure.go | 68 ----- pkg/compute/guestdrivers/base.go | 6 +- pkg/compute/guestdrivers/container.go | 4 + pkg/compute/guestdrivers/esxi.go | 114 +++++++ pkg/compute/guestdrivers/kvm.go | 6 +- pkg/compute/guestdrivers/managedvirtual.go | 99 +++++- pkg/compute/guestdrivers/qcloud.go | 68 ----- pkg/compute/guestdrivers/utils.go | 89 ++++++ pkg/compute/guestdrivers/virtualization.go | 4 - pkg/compute/hostdrivers/aliyun.go | 200 +----------- pkg/compute/hostdrivers/aws.go | 202 +----------- pkg/compute/hostdrivers/azure.go | 177 +---------- pkg/compute/hostdrivers/esxi.go | 259 ++++++++++++++++ pkg/compute/hostdrivers/kvm.go | 118 ++++--- pkg/compute/hostdrivers/managedvirtual.go | 289 ++++++++++++++++++ pkg/compute/hostdrivers/qcloud.go | 186 +---------- pkg/compute/hostdrivers/virtualization.go | 5 + pkg/compute/models/cachedimages.go | 34 +-- pkg/compute/models/cloudaccounts.go | 57 +++- pkg/compute/models/disks.go | 23 +- pkg/compute/models/guestdisks.go | 2 +- pkg/compute/models/guests.go | 6 + pkg/compute/models/hostdrivers.go | 3 + pkg/compute/models/hosts.go | 122 +++++++- pkg/compute/models/hoststorages.go | 16 +- pkg/compute/models/managedresource.go | 8 + pkg/compute/models/storagecachedimages.go | 5 +- pkg/compute/models/storagecaches.go | 30 +- pkg/compute/models/storages.go | 49 ++- pkg/compute/models/vpcs.go | 23 +- .../tasks/cloud_provider_sync_info_task.go | 6 +- ...cloud_provider_sync_onpremise_info_task.go | 14 +- pkg/compute/tasks/disk_base_task.go | 13 +- pkg/compute/tasks/disk_resize_task.go | 14 +- pkg/compute/tasks/guest_change_config_task.go | 75 ++--- pkg/compute/tasks/guest_create_disk_task.go | 101 +++++- pkg/compute/tasks/storage_cache_image_task.go | 18 +- .../tasks/storage_uncache_image_task.go | 71 ++++- pkg/mcclient/modules/managers.go | 2 +- pkg/mcclient/modules/resource.go | 2 +- pkg/util/aliyun/aliyun.go | 12 - pkg/util/aliyun/disk.go | 15 +- pkg/util/aliyun/image.go | 3 +- pkg/util/aliyun/instance.go | 6 +- pkg/util/aliyun/provider/provider.go | 22 +- pkg/util/aliyun/region.go | 34 +++ pkg/util/aliyun/storage.go | 6 +- pkg/util/aliyun/storagecache.go | 16 + pkg/util/aws/aws.go | 12 - pkg/util/aws/disk.go | 15 +- pkg/util/aws/image.go | 3 +- pkg/util/aws/instance.go | 6 +- pkg/util/aws/provider/provider.go | 18 +- pkg/util/aws/region.go | 34 +++ pkg/util/aws/storage.go | 6 +- pkg/util/aws/storagecache.go | 16 + pkg/util/azure/azure.go | 12 - pkg/util/azure/classic_disk.go | 6 +- pkg/util/azure/classic_instance.go | 11 +- pkg/util/azure/classic_storage.go | 6 +- pkg/util/azure/disk.go | 8 +- pkg/util/azure/image.go | 17 +- pkg/util/azure/instance.go | 8 +- pkg/util/azure/provider/provider.go | 18 +- pkg/util/azure/region.go | 34 +++ pkg/util/azure/storage.go | 6 +- pkg/util/azure/storagecache.go | 13 + pkg/util/esxi/datacenter.go | 38 ++- pkg/util/esxi/devtools.go | 43 +++ pkg/util/esxi/fakeregion.go | 69 +++++ pkg/util/esxi/guestos.go | 211 +++++++++++++ pkg/util/esxi/host.go | 93 +++++- pkg/util/esxi/image.go | 71 +++++ pkg/util/esxi/manager.go | 10 +- pkg/util/esxi/provider/provider.go | 50 +-- pkg/util/esxi/shell/store.go | 121 ++++++-- pkg/util/esxi/shell/virtualmachine.go | 29 ++ pkg/util/esxi/storage.go | 157 +++++++++- pkg/util/esxi/storagecache.go | 132 ++++++++ pkg/util/esxi/utils_test.go | 31 ++ pkg/util/esxi/vdisk.go | 49 ++- pkg/util/esxi/virtualmachine.go | 263 ++++++++++++++-- pkg/util/qcloud/disk.go | 8 +- pkg/util/qcloud/image.go | 3 +- pkg/util/qcloud/instance.go | 6 +- pkg/util/qcloud/localdisk.go | 4 + pkg/util/qcloud/localstorage.go | 6 +- pkg/util/qcloud/provider/provider.go | 18 +- pkg/util/qcloud/qcloud.go | 12 - pkg/util/qcloud/region.go | 34 +++ pkg/util/qcloud/storage.go | 6 +- pkg/util/qcloud/storagecache.go | 16 + 101 files changed, 3161 insertions(+), 1703 deletions(-) create mode 100644 pkg/cloudprovider/fakeregion.go create mode 100644 pkg/compute/guestdrivers/utils.go create mode 100644 pkg/compute/hostdrivers/esxi.go create mode 100644 pkg/compute/hostdrivers/managedvirtual.go create mode 100644 pkg/compute/hostdrivers/virtualization.go create mode 100644 pkg/util/esxi/devtools.go create mode 100644 pkg/util/esxi/fakeregion.go create mode 100644 pkg/util/esxi/guestos.go create mode 100644 pkg/util/esxi/image.go create mode 100644 pkg/util/esxi/storagecache.go create mode 100644 pkg/util/esxi/utils_test.go diff --git a/cmd/climc/shell/cloudmeta.go b/cmd/climc/shell/cloudmeta.go index 6fa020a639..25beae7eab 100644 --- a/cmd/climc/shell/cloudmeta.go +++ b/cmd/climc/shell/cloudmeta.go @@ -7,8 +7,8 @@ import ( func init() { type CloudmetaOptions struct { PROVIDER_ID string `help:"provider_id"` - REGION_ID string `help:"region_id"` - ZONE_ID string `help:"zone_id"` + REGION_ID string `help:"region_id"` + ZONE_ID string `help:"zone_id"` } R(&CloudmetaOptions{}, "instance-type-list", "query backend service for its version", func(s *mcclient.ClientSession, args *CloudmetaOptions) error { return nil diff --git a/pkg/cloudcommon/db/taskman/interface.go b/pkg/cloudcommon/db/taskman/interface.go index 39b56b2525..e4735c42b1 100644 --- a/pkg/cloudcommon/db/taskman/interface.go +++ b/pkg/cloudcommon/db/taskman/interface.go @@ -2,6 +2,7 @@ package taskman import ( "context" + "net/http" "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/mcclient" @@ -14,6 +15,8 @@ type ITask interface { GetTaskId() string SetStage(stageName string, data *jsonutils.JSONDict) + GetTaskRequestHeader() http.Header + SetStageComplete(ctx context.Context, data *jsonutils.JSONDict) SetStageFailed(ctx context.Context, reason string) } diff --git a/pkg/cloudcommon/db/taskman/localtaskworker.go b/pkg/cloudcommon/db/taskman/localtaskworker.go index 57eaf00fb5..1adf2d9ac0 100644 --- a/pkg/cloudcommon/db/taskman/localtaskworker.go +++ b/pkg/cloudcommon/db/taskman/localtaskworker.go @@ -18,7 +18,7 @@ func init() { func Error2TaskData(err error) jsonutils.JSONObject { errJson := jsonutils.NewDict() errJson.Add(jsonutils.NewString("ERROR"), "__status__") - errJson.Add(jsonutils.NewString(err.Error()), "reason") + errJson.Add(jsonutils.NewString(err.Error()), "__reason__") return errJson } diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index 5460fff7cc..e588bb9825 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -423,7 +423,7 @@ func execITask(taskValue reflect.Value, task *STask, odata jsonutils.JSONObject, return } - log.Debugf("Call %s %s", task.TaskName, stageName) + log.Debugf("Call %s %s %#v", task.TaskName, stageName, params) funcValue.Call(params) // call save request context @@ -648,3 +648,11 @@ func (self *STask) GetObject() db.IStandaloneModel { func (self *STask) GetObjects() []db.IStandaloneModel { return self.taskObjects } + +func (task *STask) GetTaskRequestHeader() http.Header { + header := http.Header{} + header.Set(mcclient.AUTH_TOKEN, task.GetUserCred().GetTokenString()) + header.Set(mcclient.TASK_ID, task.GetTaskId()) + header.Set(mcclient.REGION_VERSION, "v2") + return header +} diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index a464146704..ff81090e9b 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -27,12 +27,12 @@ type ICloudProvider interface { GetIRegions() []ICloudRegion GetIRegionById(id string) (ICloudRegion, error) - GetOnPremiseIHosts() ([]ICloudHost, error) + GetOnPremiseIRegion() (ICloudRegion, error) - GetIHostById(id string) (ICloudHost, error) - GetIVpcById(id string) (ICloudVpc, error) - GetIStorageById(id string) (ICloudStorage, error) - GetIStoragecacheById(id string) (ICloudStoragecache, error) + // GetIHostById(id string) (ICloudHost, error) + // GetIVpcById(id string) (ICloudVpc, error) + // GetIStorageById(id string) (ICloudStorage, error) + // GetIStoragecacheById(id string) (ICloudStoragecache, error) GetBalance() (float64, error) diff --git a/pkg/cloudprovider/fakeregion.go b/pkg/cloudprovider/fakeregion.go new file mode 100644 index 0000000000..53020b35de --- /dev/null +++ b/pkg/cloudprovider/fakeregion.go @@ -0,0 +1,76 @@ +package cloudprovider + +import ( + "yunion.io/x/jsonutils" +) + +type SFakeOnPremiseRegion struct { +} + +func (region *SFakeOnPremiseRegion) GetId() string { + return "default" +} + +func (region *SFakeOnPremiseRegion) GetName() string { + return "Default" +} + +func (region *SFakeOnPremiseRegion) GetGlobalId() string { + return "default" +} + +func (region *SFakeOnPremiseRegion) GetStatus() string { + return "available" +} + +func (region *SFakeOnPremiseRegion) Refresh() error { + return nil +} + +func (region *SFakeOnPremiseRegion) IsEmulated() bool { + return true +} + +func (region *SFakeOnPremiseRegion) GetMetadata() *jsonutils.JSONDict { + return nil +} + +func (region *SFakeOnPremiseRegion) GetLatitude() float32 { + return 0.0 +} + +func (region *SFakeOnPremiseRegion) GetLongitude() float32 { + return 0.0 +} + +func (region *SFakeOnPremiseRegion) GetIZones() ([]ICloudZone, error) { + return nil, ErrNotSupported +} + +func (region *SFakeOnPremiseRegion) GetIZoneById(id string) (ICloudZone, error) { + return nil, ErrNotSupported +} + +func (region *SFakeOnPremiseRegion) GetIVpcById(id string) (ICloudVpc, error) { + return nil, ErrNotSupported +} + +func (region *SFakeOnPremiseRegion) GetIVpcs() ([]ICloudVpc, error) { + return nil, ErrNotSupported +} + +func (region *SFakeOnPremiseRegion) GetIEips() ([]ICloudEIP, error) { + return nil, ErrNotSupported +} + +func (region *SFakeOnPremiseRegion) GetIEipById(id string) (ICloudEIP, error) { + return nil, ErrNotSupported +} + +func (region *SFakeOnPremiseRegion) CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error) { + return nil, ErrNotSupported +} + +func (region *SFakeOnPremiseRegion) CreateEIP(name string, bwMbps int, chargeType string) (ICloudEIP, error) { + return nil, ErrNotSupported +} diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 83a810f1c9..f49e7af6ef 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -36,20 +36,21 @@ type ICloudRegion interface { GetIZones() ([]ICloudZone, error) GetIVpcs() ([]ICloudVpc, error) GetIEips() ([]ICloudEIP, error) - GetISnapshots() ([]ICloudSnapshot, error) - - GetISnapshotById(snapshotId string) (ICloudSnapshot, error) - GetIZoneById(id string) (ICloudZone, error) GetIVpcById(id string) (ICloudVpc, error) - GetIHostById(id string) (ICloudHost, error) - GetIStorageById(id string) (ICloudStorage, error) - GetIStoragecacheById(id string) (ICloudStoragecache, error) + GetIZoneById(id string) (ICloudZone, error) + GetIEipById(id string) (ICloudEIP, error) CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error) - CreateEIP(name string, bwMbps int, chargeType string) (ICloudEIP, error) - GetIEipById(id string) (ICloudEIP, error) + GetISnapshots() ([]ICloudSnapshot, error) + GetISnapshotById(snapshotId string) (ICloudSnapshot, error) + + GetIHosts() ([]ICloudHost, error) + GetIHostById(id string) (ICloudHost, error) + + GetIStorages() ([]ICloudStorage, error) + GetIStorageById(id string) (ICloudStorage, error) GetProvider() string } @@ -69,7 +70,7 @@ type ICloudZone interface { type ICloudImage interface { ICloudResource - Delete() error + Delete(ctx context.Context) error GetIStoragecache() ICloudStoragecache } @@ -77,6 +78,9 @@ type ICloudStoragecache interface { ICloudResource GetIImages() ([]ICloudImage, error) + GetIImageById(extId string) (ICloudImage, error) + + GetPath() string GetManagerId() string @@ -104,7 +108,10 @@ type ICloudStorage interface { GetManagerId() string CreateIDisk(name string, sizeGb int, desc string) (ICloudDisk, error) - GetIDisk(idStr string) (ICloudDisk, error) + + GetIDiskById(idStr string) (ICloudDisk, error) + + GetMountPoint() string } type ICloudHost interface { @@ -186,10 +193,12 @@ type ICloudVM interface { DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error - ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error + ChangeConfig(ctx context.Context, ncpu int, vmem int) error GetVNCInfo() (jsonutils.JSONObject, error) AttachDisk(ctx context.Context, diskId string) error DetachDisk(ctx context.Context, diskId string) error + + CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error } type ICloudNic interface { @@ -245,13 +254,16 @@ type ICloudDisk interface { GetDriver() string GetCacheMode() string GetMountpoint() string + + GetAccessPath() string + Delete(ctx context.Context) error CreateISnapshot(ctx context.Context, name string, desc string) (ICloudSnapshot, error) GetISnapshot(idStr string) (ICloudSnapshot, error) GetISnapshots() ([]ICloudSnapshot, error) - Resize(ctx context.Context, newSize int64) error + Resize(ctx context.Context, newSizeMB int64) error Reset(ctx context.Context, snapshotId string) error } diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index 9f119b98b2..6d5ead7901 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -11,7 +11,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -96,78 +95,6 @@ func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred return data, nil } -type SDiskInfo struct { - DiskType string - Size int - Uuid string - BillingType string - FsFromat string - AutoDelete bool - TemplateId string - DiskFormat string - ExpiredAt time.Time - - Metadata map[string]string -} - -func fetchIVMinfo(desc SManagedVMCreateConfig, iVM cloudprovider.ICloudVM, guestId string, account, passwd string, action string) *jsonutils.JSONDict { - data := jsonutils.NewDict() - - data.Add(jsonutils.NewString(iVM.GetOSType()), "os") - - if len(passwd) > 0 { - encpasswd, err := utils.EncryptAESBase64(guestId, passwd) - if err != nil { - log.Errorf("encrypt password failed %s", err) - } - data.Add(jsonutils.NewString(account), "account") - data.Add(jsonutils.NewString(encpasswd), "key") - } - - if len(desc.OsDistribution) > 0 { - data.Add(jsonutils.NewString(desc.OsDistribution), "distro") - } - if len(desc.OsVersion) > 0 { - data.Add(jsonutils.NewString(desc.OsVersion), "version") - } - - idisks, err := iVM.GetIDisks() - - if err != nil { - log.Errorf("GetiDisks error %s", err) - } else { - diskInfo := make([]SDiskInfo, len(idisks)) - for i := 0; i < len(idisks); i += 1 { - dinfo := SDiskInfo{} - dinfo.Uuid = idisks[i].GetGlobalId() - dinfo.Size = idisks[i].GetDiskSizeMB() - dinfo.DiskType = idisks[i].GetDiskType() - dinfo.BillingType = idisks[i].GetBillingType() - dinfo.DiskFormat = idisks[i].GetDiskFormat() - dinfo.AutoDelete = idisks[i].GetIsAutoDelete() - if action == "create" { - dinfo.AutoDelete = true - } - dinfo.TemplateId = idisks[i].GetTemplateId() - dinfo.FsFromat = idisks[i].GetFsFormat() - dinfo.ExpiredAt = idisks[i].GetExpiredAt() - if metaData := idisks[i].GetMetadata(); metaData != nil { - dinfo.Metadata = make(map[string]string, 0) - if err := metaData.Unmarshal(dinfo.Metadata); err != nil { - log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err) - } - } - diskInfo[i] = dinfo - } - data.Add(jsonutils.Marshal(&diskInfo), "disks") - } - - data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") - data.Add(iVM.GetMetadata(), "metadata") - - return data -} - func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) log.Debugf("RequestDeployGuestOnHost: %s", config) @@ -243,65 +170,8 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return nil, err } - /*if len(guest.SecgrpId) > 0 { - if err := iVM.SyncSecurityGroup(guest.SecgrpId, guest.GetSecgroupName(), guest.GetSecRules()); err != nil { - log.Errorf("SyncSecurityGroup error: %v", err) - return nil, err - } - }*/ - - /*if onfinish == "none" { - err = iVM.StartVM() - if err != nil { - return nil, err - } - }*/ - data := fetchIVMinfo(desc, iVM, guest.Id, "root", passwd, action) - /* data.Add(jsonutils.NewString(iVM.GetOSType()), "os") - - if len(passwd) > 0 { - encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) - if err != nil { - log.Errorf("encrypt password failed %s", err) - } - data.Add(jsonutils.NewString("root"), "account") - data.Add(jsonutils.NewString(encpasswd), "key") - } - - if len(desc.OsDistribution) > 0 { - data.Add(jsonutils.NewString(desc.OsDistribution), "distro") - } - if len(desc.OsVersion) > 0 { - data.Add(jsonutils.NewString(desc.OsVersion), "version") - } - - idisks, err := iVM.GetIDisks() - - if err != nil { - log.Errorf("GetiDisks error %s", err) - } else { - diskInfo := make([]SDiskInfo, len(idisks)) - for i := 0; i < len(idisks); i += 1 { - dinfo := SDiskInfo{} - dinfo.Uuid = idisks[i].GetGlobalId() - dinfo.Size = idisks[i].GetDiskSizeMB() - if metaData := idisks[i].GetMetadata(); metaData != nil { - dinfo.Metadata = make(map[string]string, 0) - if err := metaData.Unmarshal(dinfo.Metadata); err != nil { - log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err) - } - } - diskInfo[i] = dinfo - } - data.Add(jsonutils.Marshal(&diskInfo), "disks") - } - - data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") - data.Add(iVM.GetMetadata(), "metadata") - */ - return data, nil }) } else if action == "deploy" { @@ -346,20 +216,6 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu data := fetchIVMinfo(desc, iVM, guest.Id, "root", passwd, action) - /* - data := jsonutils.NewDict() - - if len(passwd) > 0 { - encpasswd, err := utils.EncryptAESBase64(guest.Id, passwd) - if err != nil { - log.Errorf("encrypt password failed %s", err) - } - - - data.Add(jsonutils.NewString("root"), "account") // 用户名 - data.Add(jsonutils.NewString(encpasswd), "key") // 密码 - }*/ - return data, nil }) } else if action == "rebuild" { @@ -432,75 +288,6 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return nil } -func (self *SAliyunGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { - - if data.Contains("disks") { - diskInfo := make([]SDiskInfo, 0) - err := data.Unmarshal(&diskInfo, "disks") - if err != nil { - return err - } - disks := guest.GetDisks() - if len(disks) != len(diskInfo) { - msg := fmt.Sprintf("inconsistent disk number: have %d want %d", len(disks), len(diskInfo)) - log.Errorf(msg) - return fmt.Errorf(msg) - } - for i := 0; i < len(diskInfo); i += 1 { - disk := disks[i].GetDisk() - _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.DiskSize = diskInfo[i].Size - disk.ExternalId = diskInfo[i].Uuid - disk.DiskType = diskInfo[i].DiskType - disk.Status = models.DISK_READY - disk.BillingType = diskInfo[i].BillingType - disk.FsFormat = diskInfo[i].FsFromat - if diskInfo[i].AutoDelete { - disk.AutoDelete = true - } - // disk.TemplateId = diskInfo[i].TemplateId - disk.DiskFormat = diskInfo[i].DiskFormat - disk.ExpiredAt = diskInfo[i].ExpiredAt - if len(diskInfo[i].Metadata) > 0 { - for key, value := range diskInfo[i].Metadata { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - return nil - }) - if err != nil { - msg := fmt.Sprintf("save disk info failed %s", err) - log.Errorf(msg) - break - } else { - db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), task.GetUserCred()) - } - } - } - uuid, _ := data.GetString("uuid") - if len(uuid) > 0 { - guest.SetExternalId(uuid) - } - - if metaData, _ := data.Get("metadata"); metaData != nil { - meta := make(map[string]string, 0) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get guest %s metadata error: %v", guest.Name, err) - } else { - for key, value := range meta { - if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) - } - } - } - } - - guest.SaveDeployInfo(ctx, task.GetUserCred(), data) - return nil -} - func (self *SAliyunGuestDriver) AllowReconfigGuest() bool { return true } diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index 0a9f101397..d0ecf9d556 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -9,7 +9,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -20,6 +19,11 @@ type SAwsGuestDriver struct { SManagedVirtualizedGuestDriver } +func init() { + driver := SAwsGuestDriver{} + models.RegisterGuestDriver(&driver) +} + func (self *SAwsGuestDriver) GetHypervisor() string { return models.HYPERVISOR_AWS } @@ -223,74 +227,3 @@ func (self *SAwsGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest return nil } - -func (self *SAwsGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { - if data.Contains("disks") { - diskInfo := make([]SDiskInfo, 0) - err := data.Unmarshal(&diskInfo, "disks") - if err != nil { - return err - } - disks := guest.GetDisks() - if len(disks) != len(diskInfo) { - msg := fmt.Sprintf("inconsistent disk number: have %d want %d", len(disks), len(diskInfo)) - log.Errorf(msg) - return fmt.Errorf(msg) - } - for i := 0; i < len(diskInfo); i += 1 { - disk := disks[i].GetDisk() - _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.DiskSize = diskInfo[i].Size - disk.ExternalId = diskInfo[i].Uuid - disk.DiskType = diskInfo[i].DiskType - disk.Status = models.DISK_READY - disk.BillingType = diskInfo[i].BillingType - disk.FsFormat = diskInfo[i].FsFromat - disk.AutoDelete = true - //disk.TemplateId = diskInfo[i].TemplateId - disk.DiskFormat = diskInfo[i].DiskFormat - disk.ExpiredAt = diskInfo[i].ExpiredAt - if len(diskInfo[i].Metadata) > 0 { - for key, value := range diskInfo[i].Metadata { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - return nil - }) - if err != nil { - msg := fmt.Sprintf("save disk info failed %s", err) - log.Errorf(msg) - break - } else { - db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), task.GetUserCred()) - } - } - } - uuid, _ := data.GetString("uuid") - if len(uuid) > 0 { - guest.SetExternalId(uuid) - } - - if metaData, _ := data.Get("metadata"); metaData != nil { - meta := make(map[string]string, 0) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get guest %s metadata error: %v", guest.Name, err) - } else { - for key, value := range meta { - if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) - } - } - } - } - - guest.SaveDeployInfo(ctx, task.GetUserCred(), data) - return nil -} - -func init() { - driver := SAwsGuestDriver{} - models.RegisterGuestDriver(&driver) -} diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index 695544f7be..09541353cb 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -14,7 +14,6 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -216,70 +215,3 @@ func (self *SAzureGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gue } return nil } - -func (self *SAzureGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { - - if data.Contains("disks") { - diskInfo := make([]SDiskInfo, 0) - err := data.Unmarshal(&diskInfo, "disks") - if err != nil { - return err - } - disks := guest.GetDisks() - if len(disks) != len(diskInfo) { - msg := fmt.Sprintf("inconsistent disk number: have %d want %d", len(disks), len(diskInfo)) - log.Errorf(msg) - return fmt.Errorf(msg) - } - for i := 0; i < len(diskInfo); i++ { - disk := disks[i].GetDisk() - _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.DiskSize = diskInfo[i].Size - disk.ExternalId = diskInfo[i].Uuid - disk.DiskType = diskInfo[i].DiskType - disk.Status = models.DISK_READY - disk.BillingType = diskInfo[i].BillingType - disk.FsFormat = diskInfo[i].FsFromat - disk.AutoDelete = diskInfo[i].AutoDelete - // disk.TemplateId = diskInfo[i].TemplateId - disk.DiskFormat = diskInfo[i].DiskFormat - disk.ExpiredAt = diskInfo[i].ExpiredAt - if len(diskInfo[i].Metadata) > 0 { - for key, value := range diskInfo[i].Metadata { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - return nil - }) - if err != nil { - msg := fmt.Sprintf("save disk info failed %s", err) - log.Errorf(msg) - break - } else { - db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), task.GetUserCred()) - } - } - } - uuid, _ := data.GetString("uuid") - if len(uuid) > 0 { - guest.SetExternalId(uuid) - } - - if metaData, _ := data.Get("metadata"); metaData != nil { - meta := make(map[string]string, 0) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get guest %s metadata error: %v", guest.Name, err) - } else { - for key, value := range meta { - if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) - } - } - } - } - - guest.SaveDeployInfo(ctx, task.GetUserCred(), data) - return nil -} diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 8d9207b7c2..a406790d43 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -185,9 +185,5 @@ func (self *SBaseGuestDriver) RequestReloadDiskSnapshot(ctx context.Context, gue } func (self *SBaseGuestDriver) getTaskRequestHeader(task taskman.ITask) http.Header { - header := http.Header{} - header.Set(mcclient.AUTH_TOKEN, task.GetUserCred().GetTokenString()) - header.Set(mcclient.TASK_ID, task.GetTaskId()) - header.Set(mcclient.REGION_VERSION, "v2") - return header + return task.GetTaskRequestHeader() } diff --git a/pkg/compute/guestdrivers/container.go b/pkg/compute/guestdrivers/container.go index 3d57546971..0a5b56e406 100644 --- a/pkg/compute/guestdrivers/container.go +++ b/pkg/compute/guestdrivers/container.go @@ -112,6 +112,10 @@ func (self *SContainerDriver) OnGuestDeployTaskComplete(ctx context.Context, gue return nil } +func (self *SContainerDriver) GetJsonDescAtHost(ctx context.Context, guest *models.SGuest, host *models.SHost) jsonutils.JSONObject { + return guest.GetJsonDescAtHypervisor(ctx, host) +} + func (self *SContainerDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) config.Add(jsonutils.JSONTrue, "k8s_pod") diff --git a/pkg/compute/guestdrivers/esxi.go b/pkg/compute/guestdrivers/esxi.go index 7d06890ba5..98010911a3 100644 --- a/pkg/compute/guestdrivers/esxi.go +++ b/pkg/compute/guestdrivers/esxi.go @@ -2,6 +2,12 @@ package guestdrivers import ( "context" + "fmt" + "net/http" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -33,6 +39,10 @@ func (self *SESXiGuestDriver) GetAttachDiskStatus() ([]string, error) { return []string{models.VM_READY}, nil } +func (self *SESXiGuestDriver) GetChangeConfigStatus() ([]string, error) { + return []string{models.VM_READY}, nil +} + func (self *SESXiGuestDriver) CanKeepDetachDisk() bool { return false } @@ -50,3 +60,107 @@ func (self *SESXiGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest * task.ScheduleRun(nil) return nil } + +func (self *SESXiGuestDriver) GetRebuildRootStatus() ([]string, error) { + return []string{models.VM_READY}, nil +} + +func (self *SESXiGuestDriver) GetDeployStatus() ([]string, error) { + return []string{models.VM_READY}, nil +} + +func (self *SESXiGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { + if !utils.IsInStringArray(guest.Status, []string{models.VM_READY}) { + return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status) + } + if disk.DiskType == models.DISK_TYPE_SYS { + return fmt.Errorf("Cannot resize system disk") + } + /*if !utils.IsInStringArray(storage.StorageType, []string{models.STORAGE_PUBLIC_CLOUD, models.STORAGE_CLOUD_SSD, models.STORAGE_CLOUD_EFFICIENCY}) { + return fmt.Errorf("Cannot resize %s disk", storage.StorageType) + }*/ + return nil +} + +func (self *SESXiGuestDriver) RequestDetachDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) +} + +func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *models.SGuest, host *models.SHost) jsonutils.JSONObject { + return guest.GetJsonDescAtHypervisor(ctx, host) +} + +func (self *SESXiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { + config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) + log.Debugf("RequestDeployGuestOnHost: %s", config) + + agent, err := host.GetEsxiAgentHost() + if err != nil { + return err + } + if agent == nil { + return fmt.Errorf("No ESXi agent host") + } + + diskCat := guest.CategorizeDisks() + if diskCat.Root == nil { + return fmt.Errorf("no root disk???") + } + storage := diskCat.Root.GetStorage() + if storage == nil { + return fmt.Errorf("root disk has no storage???") + } + + config.Add(jsonutils.NewString(host.AccessIp), "host_ip") + config.Add(jsonutils.NewString(guest.Id), "guest_id") + + accessInfo, err := host.GetCloudaccount().GetVCenterAccessInfo(storage.ExternalId) + if err != nil { + return err + } + config.Add(jsonutils.Marshal(accessInfo), "datastore") + + url := "/disks/agent/deploy" + + body := jsonutils.NewDict() + body.Add(config, "disk") + + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + + _, err = agent.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SESXiGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { + + if data.Contains("host_ip") { + hostIp, _ := data.GetString("host_ip") + host, err := models.HostManager.GetHostByIp(hostIp) + if err != nil { + log.Errorf("fail to find host with IP %s: %s", hostIp, err) + return err + } + if host.Id != guest.HostId { + models.HostManager.ClearSchedDescCache(host.Id) + models.HostManager.ClearSchedDescCache(guest.HostId) + guest.SetHostId(host.Id) + } + } + + return self.SManagedVirtualizedGuestDriver.OnGuestDeployTaskDataReceived(ctx, guest, task, data) +} + +func (self *SESXiGuestDriver) AllowReconfigGuest() bool { + return true +} + +func (self *SESXiGuestDriver) DoGuestCreateDisksTask(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "ESXiGuestCreateDiskTask", guest, task.GetUserCred(), task.GetParams(), task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 1cb9d0d586..4c3be6b1cd 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -179,6 +179,10 @@ func (self *SKVMGuestDriver) RequestUndeployGuestOnHost(ctx context.Context, gue return nil } +func (self *SKVMGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *models.SGuest, host *models.SHost) jsonutils.JSONObject { + return guest.GetJsonDescAtHypervisor(ctx, host) +} + func (self *SKVMGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) log.Debugf("RequestDeployGuestOnHost: %s", config) @@ -207,7 +211,7 @@ func (self *SKVMGuestDriver) RequestStartOnHost(ctx context.Context, guest *mode header := self.getTaskRequestHeader(task) config := jsonutils.NewDict() - desc := self.GetJsonDescAtHost(ctx, guest, host) + desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host) config.Add(desc, "desc") params := task.GetParams() if params.Length() > 0 { diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 4d0c3aa63d..9b28e82b0a 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/secrules" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -116,10 +117,6 @@ func (self *SManagedVirtualizedGuestDriver) RequestDeployGuestOnHost(ctx context return nil } -func (self *SManagedVirtualizedGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { - return nil -} - func (self *SManagedVirtualizedGuestDriver) RequestStartOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential, task taskman.ITask) (jsonutils.JSONObject, error) { ihost, e := host.GetIHost() if e != nil { @@ -149,18 +146,20 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { ihost, err := host.GetIHost() if err != nil { + log.Errorf("host.GetIHost fail %s", err) return nil, err } ivm, err := ihost.GetIVMById(guest.ExternalId) if err != nil { if err == cloudprovider.ErrNotFound { return nil, nil - } else { - return nil, err } + log.Errorf("ihost.GetIVMById fail %s", err) + return nil, err } err = ivm.DeleteVM(ctx) if err != nil { + log.Errorf("ivm.DeleteVM fail %s", err) return nil, err } @@ -170,12 +169,13 @@ func (self *SManagedVirtualizedGuestDriver) RequestUndeployGuestOnHost(ctx conte if err != nil { if err == cloudprovider.ErrNotFound { continue - } else { - return nil, err } + log.Errorf("disk.GetIDisk fail %s", err) + return nil, err } err = idisk.Delete(ctx) if err != nil { + log.Errorf("idisk.Delete fail %s", err) return nil, err } } @@ -297,7 +297,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestChangeVmConfig(ctx context.Co } if int(guest.VcpuCount) != config.Cpu || guest.VmemSize != config.Memory { - err = iVM.ChangeConfig(ctx, config.InstanceId, config.Cpu, config.Memory) + err = iVM.ChangeConfig(ctx, config.Cpu, config.Memory) if err != nil { return err } @@ -372,3 +372,84 @@ func (self *SManagedVirtualizedGuestDriver) RequestDiskSnapshot(ctx context.Cont }) return nil } + +func (self *SManagedVirtualizedGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { + + if data.Contains("disks") { + diskInfo := make([]SDiskInfo, 0) + err := data.Unmarshal(&diskInfo, "disks") + if err != nil { + return err + } + + disks := guest.GetDisks() + if len(disks) != len(diskInfo) { + msg := fmt.Sprintf("inconsistent disk number: have %d want %d", len(disks), len(diskInfo)) + log.Errorf(msg) + return fmt.Errorf(msg) + } + for i := 0; i < len(diskInfo); i += 1 { + disk := disks[i].GetDisk() + _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.DiskSize = diskInfo[i].Size + disk.ExternalId = diskInfo[i].Uuid + disk.DiskType = diskInfo[i].DiskType + disk.Status = models.DISK_READY + disk.BillingType = diskInfo[i].BillingType + disk.FsFormat = diskInfo[i].FsFromat + if diskInfo[i].AutoDelete { + disk.AutoDelete = true + } + // disk.TemplateId = diskInfo[i].TemplateId + disk.AccessPath = diskInfo[i].Path + disk.DiskFormat = diskInfo[i].DiskFormat + disk.ExpiredAt = diskInfo[i].ExpiredAt + if len(diskInfo[i].Metadata) > 0 { + for key, value := range diskInfo[i].Metadata { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + return nil + }) + if err != nil { + msg := fmt.Sprintf("save disk info failed %s", err) + log.Errorf(msg) + break + } + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), task.GetUserCred()) + guestdisk := guest.GetGuestDisk(disk.Id) + _, err = guestdisk.GetModelManager().TableSpec().Update(guestdisk, func() error { + guestdisk.Driver = diskInfo[i].Driver + guestdisk.CacheMode = diskInfo[i].CacheMode + return nil + }) + if err != nil { + msg := fmt.Sprintf("save disk info failed %s", err) + log.Errorf(msg) + break + } + } + } + uuid, _ := data.GetString("uuid") + if len(uuid) > 0 { + guest.SetExternalId(uuid) + } + + if metaData, _ := data.Get("metadata"); metaData != nil { + meta := make(map[string]string, 0) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get guest %s metadata error: %v", guest.Name, err) + } else { + for key, value := range meta { + if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) + } + } + } + } + + guest.SaveDeployInfo(ctx, task.GetUserCred(), data) + return nil +} diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index 10eee31d5b..baa50e286c 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -12,7 +12,6 @@ import ( "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/utils" - "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" @@ -275,73 +274,6 @@ func (self *SQcloudGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu return nil } -func (self *SQcloudGuestDriver) OnGuestDeployTaskDataReceived(ctx context.Context, guest *models.SGuest, task taskman.ITask, data jsonutils.JSONObject) error { - - if data.Contains("disks") { - diskInfo := make([]SDiskInfo, 0) - err := data.Unmarshal(&diskInfo, "disks") - if err != nil { - return err - } - disks := guest.GetDisks() - if len(disks) != len(diskInfo) { - msg := fmt.Sprintf("inconsistent disk number: have %d want %d", len(disks), len(diskInfo)) - log.Errorf(msg) - return fmt.Errorf(msg) - } - for i := 0; i < len(diskInfo); i += 1 { - disk := disks[i].GetDisk() - _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.DiskSize = diskInfo[i].Size - disk.ExternalId = diskInfo[i].Uuid - disk.DiskType = diskInfo[i].DiskType - disk.Status = models.DISK_READY - disk.BillingType = diskInfo[i].BillingType - disk.FsFormat = diskInfo[i].FsFromat - disk.AutoDelete = true - //disk.TemplateId = diskInfo[i].TemplateId - disk.DiskFormat = diskInfo[i].DiskFormat - disk.ExpiredAt = diskInfo[i].ExpiredAt - if len(diskInfo[i].Metadata) > 0 { - for key, value := range diskInfo[i].Metadata { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - return nil - }) - if err != nil { - msg := fmt.Sprintf("save disk info failed %s", err) - log.Errorf(msg) - break - } else { - db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), task.GetUserCred()) - } - } - } - uuid, _ := data.GetString("uuid") - if len(uuid) > 0 { - guest.SetExternalId(uuid) - } - - if metaData, _ := data.Get("metadata"); metaData != nil { - meta := make(map[string]string, 0) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get guest %s metadata error: %v", guest.Name, err) - } else { - for key, value := range meta { - if err := guest.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set guest %s mata %s => %s error: %v", guest.Name, key, value, err) - } - } - } - } - - guest.SaveDeployInfo(ctx, task.GetUserCred(), data) - return nil -} - func (self *SQcloudGuestDriver) AllowReconfigGuest() bool { return true } diff --git a/pkg/compute/guestdrivers/utils.go b/pkg/compute/guestdrivers/utils.go new file mode 100644 index 0000000000..ca594446ce --- /dev/null +++ b/pkg/compute/guestdrivers/utils.go @@ -0,0 +1,89 @@ +package guestdrivers + +import ( + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SDiskInfo struct { + DiskType string + Size int + Uuid string + BillingType string + FsFromat string + AutoDelete bool + TemplateId string + DiskFormat string + Path string + Driver string + CacheMode string + ExpiredAt time.Time + + Metadata map[string]string +} + +func fetchIVMinfo(desc SManagedVMCreateConfig, iVM cloudprovider.ICloudVM, guestId string, account, passwd string, action string) *jsonutils.JSONDict { + data := jsonutils.NewDict() + + data.Add(jsonutils.NewString(iVM.GetOSType()), "os") + + if len(passwd) > 0 { + encpasswd, err := utils.EncryptAESBase64(guestId, passwd) + if err != nil { + log.Errorf("encrypt password failed %s", err) + } + data.Add(jsonutils.NewString(account), "account") + data.Add(jsonutils.NewString(encpasswd), "key") + } + + if len(desc.OsDistribution) > 0 { + data.Add(jsonutils.NewString(desc.OsDistribution), "distro") + } + if len(desc.OsVersion) > 0 { + data.Add(jsonutils.NewString(desc.OsVersion), "version") + } + + idisks, err := iVM.GetIDisks() + + if err != nil { + log.Errorf("GetiDisks error %s", err) + } else { + diskInfo := make([]SDiskInfo, len(idisks)) + for i := 0; i < len(idisks); i += 1 { + dinfo := SDiskInfo{} + dinfo.Uuid = idisks[i].GetGlobalId() + dinfo.Size = idisks[i].GetDiskSizeMB() + dinfo.DiskType = idisks[i].GetDiskType() + dinfo.BillingType = idisks[i].GetBillingType() + dinfo.DiskFormat = idisks[i].GetDiskFormat() + dinfo.AutoDelete = idisks[i].GetIsAutoDelete() + if action == "create" { + dinfo.AutoDelete = true + } + dinfo.Path = idisks[i].GetAccessPath() + dinfo.Driver = idisks[i].GetDriver() + dinfo.CacheMode = idisks[i].GetCacheMode() + dinfo.TemplateId = idisks[i].GetTemplateId() + dinfo.FsFromat = idisks[i].GetFsFormat() + dinfo.ExpiredAt = idisks[i].GetExpiredAt() + if metaData := idisks[i].GetMetadata(); metaData != nil { + dinfo.Metadata = make(map[string]string, 0) + if err := metaData.Unmarshal(dinfo.Metadata); err != nil { + log.Errorf("Get disk %s metadata info error: %v", idisks[i].GetName(), err) + } + } + diskInfo[i] = dinfo + } + data.Add(jsonutils.Marshal(&diskInfo), "disks") + } + + data.Add(jsonutils.NewString(iVM.GetGlobalId()), "uuid") + data.Add(iVM.GetMetadata(), "metadata") + + return data +} diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index 51ba216d1b..dd9aff8208 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -185,10 +185,6 @@ func (self *SVirtualizedGuestDriver) ValidateCreateHostData(ctx context.Context, return data, nil } -func (self *SVirtualizedGuestDriver) GetJsonDescAtHost(ctx context.Context, guest *models.SGuest, host *models.SHost) jsonutils.JSONObject { - return guest.GetJsonDescAtHypervisor(ctx, host) -} - func (self *SVirtualizedGuestDriver) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, data *jsonutils.JSONDict) error { return guest.StartGueststartTask(ctx, userCred, data, "") } diff --git a/pkg/compute/hostdrivers/aliyun.go b/pkg/compute/hostdrivers/aliyun.go index bff39e84af..6d7fc57c7d 100644 --- a/pkg/compute/hostdrivers/aliyun.go +++ b/pkg/compute/hostdrivers/aliyun.go @@ -1,22 +1,11 @@ package hostdrivers import ( - "context" - "fmt" - "os" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/compute/options" - "yunion.io/x/onecloud/pkg/httperrors" ) type SAliyunHostDriver struct { - SBaseHostDriver + SManagedVirtualizationHostDriver } func init() { @@ -27,190 +16,3 @@ func init() { func (self *SAliyunHostDriver) GetHostType() string { return models.HOST_TYPE_ALIYUN } - -func (self *SAliyunHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { - params := task.GetParams() - imageId, err := params.GetString("image_id") - if err != nil { - return err - } - - osArch, _ := params.GetString("os_arch") - osType, _ := params.GetString("os_type") - osDist, _ := params.GetString("os_distribution") - - isForce := jsonutils.QueryBoolean(params, "is_force", false) - userCred := task.GetUserCred() - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - - lockman.LockRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - defer lockman.ReleaseRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), storageCache.Id, imageId) - - iStorageCache, err := storageCache.GetIStorageCache() - if err != nil { - return nil, err - } - - extImgId, err := iStorageCache.UploadImage(userCred, imageId, osArch, osType, osDist, scimg.ExternalId, isForce) - - if err != nil { - return nil, err - } else { - scimg.SetExternalId(extImgId) - - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(extImgId), "image_id") - return ret, nil - } - }) - return nil -} - -func (self *SAliyunHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error { - task.ScheduleRun(nil) - return nil -} - -func (self *SAliyunHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error { - if iDisk, err := disk.GetIDisk(); err != nil { - return err - } else if iStorage, err := disk.GetIStorage(); err != nil { - return err - } else if iStoragecache := iStorage.GetIStoragecache(); iStoragecache == nil { - return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) - } else { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - if snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { - return nil, err - } else { - params := task.GetParams() - osType, _ := params.GetString("properties", "os_type") - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId) - if scimg.Status != models.CACHED_IMAGE_STATUS_READY { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host") - } - if iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, ""); err != nil { - log.Errorf("fail to create iImage: %v", err) - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } else { - scimg.SetExternalId(iImage.GetId()) - if _, err := os.Stat(options.Options.TempPath); os.IsNotExist(err) { - if err = os.MkdirAll(options.Options.TempPath, 0755); err != nil { - return nil, err - } - } - if result, err := iStoragecache.DownloadImage(task.GetUserCred(), imageId, iImage.GetId(), options.Options.TempPath); err != nil { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } else { - if err := iImage.Delete(); err != nil { - log.Errorf("Delete iImage %s failed: %v", iImage.GetId(), err) - } - if err := snapshot.Delete(); err != nil { - log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err) - } - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_READY, "") - return result, nil - } - } - } - }) - } - return nil -} - -func (self *SAliyunHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else { - if size, err := content.Int("size"); err != nil { - return err - } else { - size = size >> 10 - if iDisk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), ""); err != nil { - return err - } else { - if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.ExternalId = iDisk.GetGlobalId() - - if metaData := iDisk.GetMetadata(); metaData != nil { - meta := make(map[string]string) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) - } else { - for key, value := range meta { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - } - - return nil - }); err != nil { - log.Errorf("Update disk externalId err: %v", err) - return err - } - data := jsonutils.NewDict() - data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") - data.Add(jsonutils.NewString(iDisk.GetDiskFormat()), "disk_format") - task.ScheduleRun(data) - } - } - } - return nil -} - -func (self *SAliyunHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { - data := jsonutils.NewDict() - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { - if err == cloudprovider.ErrNotFound { - task.ScheduleRun(data) - return nil - } - return err - } else if err := iDisk.Delete(ctx); err != nil { - return err - } - task.ScheduleRun(data) - return nil -} - -func (self *SAliyunHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) -} - -func (self *SAliyunHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { - return err - } else if err := iDisk.Resize(ctx, size>>10); err != nil { - return err - } else { - task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) - } - return nil -} - -func (self *SAliyunHostDriver) RequestResetDisk(ctx context.Context, host *models.SHost, disk *models.SDisk, params *jsonutils.JSONDict, task taskman.ITask) error { - iDisk, err := disk.GetIDisk() - if err != nil { - return err - } - snapshotId, err := params.GetString("snapshot_id") - if err != nil { - return err - } - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - err := iDisk.Reset(ctx, snapshotId) - return nil, err - }) - return nil -} diff --git a/pkg/compute/hostdrivers/aws.go b/pkg/compute/hostdrivers/aws.go index c482cc4086..4a7fa74563 100644 --- a/pkg/compute/hostdrivers/aws.go +++ b/pkg/compute/hostdrivers/aws.go @@ -1,210 +1,18 @@ package hostdrivers import ( - "context" - "fmt" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/options" - "yunion.io/x/onecloud/pkg/httperrors" - - "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" ) type SAwsHostDriver struct { - SBaseHostDriver -} - -func (self *SAwsHostDriver) GetHostType() string { - return models.HOST_TYPE_AWS -} - -func (self *SAwsHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { - params := task.GetParams() - imageId, err := params.GetString("image_id") - if err != nil { - return err - } - - osArch, _ := params.GetString("os_arch") - osType, _ := params.GetString("os_type") - osDist, _ := params.GetString("os_distribution") - - isForce := jsonutils.QueryBoolean(params, "is_force", false) - userCred := task.GetUserCred() - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - - lockman.LockRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - defer lockman.ReleaseRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), storageCache.Id, imageId) - - iStorageCache, err := storageCache.GetIStorageCache() - if err != nil { - return nil, err - } - - extImgId, err := iStorageCache.UploadImage(userCred, imageId, osArch, osType, osDist, scimg.ExternalId, isForce) - - if err != nil { - return nil, err - } else { - scimg.SetExternalId(extImgId) - - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(extImgId), "image_id") - return ret, nil - } - }) - return nil -} - -func (self *SAwsHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error { - task.ScheduleRun(nil) - return nil -} - -func (self *SAwsHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error { - if iDisk, err := disk.GetIDisk(); err != nil { - return err - } else if iStorage, err := disk.GetIStorage(); err != nil { - return err - } else if iStoragecache := iStorage.GetIStoragecache(); iStoragecache == nil { - return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) - } else { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - if snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { - return nil, err - } else { - params := task.GetParams() - osType, _ := params.GetString("properties", "os_type") - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId) - if scimg.Status != models.CACHED_IMAGE_STATUS_READY { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host") - } - if iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, ""); err != nil { - log.Errorf("fail to create iImage: %v", err) - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } else { - scimg.SetExternalId(iImage.GetId()) - if result, err := iStoragecache.DownloadImage(task.GetUserCred(), imageId, iImage.GetId(), options.Options.TempPath); err != nil { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } else { - if err := iImage.Delete(); err != nil { - log.Errorf("Delete iImage %s failed: %v", iImage.GetId(), err) - } - if err := snapshot.Delete(); err != nil { - log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err) - } - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_READY, "") - return result, nil - } - } - } - }) - } - return nil -} - -func (self *SAwsHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else { - if size, err := content.Int("size"); err != nil { - return err - } else { - size = size >> 10 - if iDisk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), ""); err != nil { - return err - } else { - if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.ExternalId = iDisk.GetGlobalId() - - if metaData := iDisk.GetMetadata(); metaData != nil { - meta := make(map[string]string) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) - } else { - for key, value := range meta { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - } - - return nil - }); err != nil { - log.Errorf("Update disk externalId err: %v", err) - return err - } - data := jsonutils.NewDict() - data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") - data.Add(jsonutils.NewString(iDisk.GetDiskFormat()), "disk_format") - task.ScheduleRun(data) - } - } - } - return nil -} - -func (self *SAwsHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { - data := jsonutils.NewDict() - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { - if err == cloudprovider.ErrNotFound { - task.ScheduleRun(data) - return nil - } - return err - } else if err := iDisk.Delete(ctx); err != nil { - return err - } - task.ScheduleRun(data) - return nil -} - -func (self *SAwsHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) -} - -func (self *SAwsHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { - return err - } else if err := iDisk.Resize(ctx, size>>10); err != nil { - return err - } else { - task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) - } - return nil -} - -func (self *SAwsHostDriver) RequestResetDisk(ctx context.Context, host *models.SHost, disk *models.SDisk, params *jsonutils.JSONDict, task taskman.ITask) error { - iDisk, err := disk.GetIDisk() - if err != nil { - return err - } - snapshotId, err := params.GetString("snapshot_id") - if err != nil { - return err - } - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - err := iDisk.Reset(snapshotId) - return nil, err - }) - return nil + SManagedVirtualizationHostDriver } func init() { driver := SAwsHostDriver{} models.RegisterHostDriver(&driver) } + +func (self *SAwsHostDriver) GetHostType() string { + return models.HOST_TYPE_AWS +} diff --git a/pkg/compute/hostdrivers/azure.go b/pkg/compute/hostdrivers/azure.go index 93581da592..d2552d309c 100644 --- a/pkg/compute/hostdrivers/azure.go +++ b/pkg/compute/hostdrivers/azure.go @@ -2,22 +2,15 @@ package hostdrivers import ( "context" - "fmt" - "os" - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) type SAzureHostDriver struct { - SBaseHostDriver + SManagedVirtualizationHostDriver } func init() { @@ -36,174 +29,6 @@ func (self *SAzureHostDriver) ValidateUpdateDisk(ctx context.Context, userCred m return data, nil } -func (self *SAzureHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { - params := task.GetParams() - imageId, err := params.GetString("image_id") - if err != nil { - return err - } - - osArch, _ := params.GetString("os_arch") - osType, _ := params.GetString("os_type") - osDist, _ := params.GetString("os_distribution") - - isForce := jsonutils.QueryBoolean(params, "is_force", false) - userCred := task.GetUserCred() - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - lockman.LockRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - defer lockman.ReleaseRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), storageCache.Id, imageId) - iStorageCache, err := storageCache.GetIStorageCache() - if err != nil { - return nil, err - } - - extImgId, err := iStorageCache.UploadImage(userCred, imageId, osArch, osType, osDist, scimg.ExternalId, isForce) - if err != nil { - return nil, err - } else { - scimg.SetExternalId(extImgId) - - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(extImgId), "image_id") - return ret, nil - } - }) - return nil -} - -func (self *SAzureHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else { - if size, err := content.Int("size"); err != nil { - return err - } else { - size = size >> 10 - if iDisk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), ""); err != nil { - return err - } else { - if _, err := disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.ExternalId = iDisk.GetGlobalId() - - if metaData := iDisk.GetMetadata(); metaData != nil { - meta := make(map[string]string) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) - } else { - for key, value := range meta { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - } - - return nil - }); err != nil { - log.Errorf("Update disk externalId err: %v", err) - return err - } - data := jsonutils.NewDict() - data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") - data.Add(jsonutils.NewString(iDisk.GetDiskFormat()), "disk_format") - task.ScheduleRun(data) - } - } - } - return nil -} - -func (self *SAzureHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { - data := jsonutils.NewDict() - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { - if err == cloudprovider.ErrNotFound { - task.ScheduleRun(data) - return nil - } - return err - } else if err := iDisk.Delete(ctx); err != nil { - return err - } - task.ScheduleRun(data) - return nil -} - -func (self *SAzureHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) -} - -func (self *SAzureHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { - return err - } else if err := iDisk.Resize(ctx, size>>10); err != nil { - return err - } else { - task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) - } - return nil -} - -func (self *SAzureHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error { - task.ScheduleRun(nil) - return nil -} - -func (self *SAzureHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error { - if iDisk, err := disk.GetIDisk(); err != nil { - return err - } else if iStorage, err := disk.GetIStorage(); err != nil { - return err - } else if iStoragecache := iStorage.GetIStoragecache(); iStoragecache == nil { - return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) - } else { - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - if snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage"); err != nil { - return nil, err - } else { - params := task.GetParams() - osType, _ := params.GetString("properties", "os_type") - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId) - if scimg.Status != models.CACHED_IMAGE_STATUS_READY { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host") - } - if iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, ""); err != nil { - log.Errorf("fail to create iImage: %v", err) - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } else { - scimg.SetExternalId(iImage.GetId()) - if _, err := os.Stat(options.Options.TempPath); os.IsNotExist(err) { - if err = os.MkdirAll(options.Options.TempPath, 0755); err != nil { - return nil, err - } - } - if result, err := iStoragecache.DownloadImage(task.GetUserCred(), imageId, iImage.GetId(), options.Options.TempPath); err != nil { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } else { - if err := iImage.Delete(); err != nil { - log.Errorf("Delete iImage %s failed: %v", iImage.GetId(), err) - } - if err := snapshot.Delete(); err != nil { - log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err) - } - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_READY, "") - return result, nil - } - } - } - }) - } - return nil -} - func (self *SAzureHostDriver) RequestDeleteSnapshotWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error { return httperrors.NewNotImplementedError("not implement") } diff --git a/pkg/compute/hostdrivers/esxi.go b/pkg/compute/hostdrivers/esxi.go new file mode 100644 index 0000000000..d7d1158007 --- /dev/null +++ b/pkg/compute/hostdrivers/esxi.go @@ -0,0 +1,259 @@ +package hostdrivers + +import ( + "context" + "fmt" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +type SESXiHostDriver struct { + SManagedVirtualizationHostDriver +} + +func init() { + driver := SESXiHostDriver{} + models.RegisterHostDriver(&driver) +} + +func (self *SESXiHostDriver) GetHostType() string { + return models.HOST_TYPE_ESXI +} + +func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + isForce := jsonutils.QueryBoolean(params, "is_force", false) + obj, err := models.CachedimageManager.FetchById(imageId) + if err != nil { + return err + } + cacheImage := obj.(*models.SCachedimage) + srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(models.HOST_TYPE_ESXI, []string{host.Id}, + []interface{}{host.GetZone(), host.GetCloudprovider()}) + if err != nil { + return err + } + + type contentStruct struct { + ImageId string + HostId string + HostIp string + SrcHostIp string + SrcPath string + SrcDatastore models.SVCenterAccessInfo + Datastore models.SVCenterAccessInfo + Format string + IsForce bool + StoragecacheId string + } + + content := contentStruct{} + content.ImageId = imageId + content.HostId = host.Id + content.HostIp = host.AccessIp + content.Format = cacheImage.GetFormat() + + storage := host.GetStorageByFilePath(storageCache.Path) + accessInfo, err := host.GetCloudaccount().GetVCenterAccessInfo(storage.ExternalId) + if err != nil { + return err + } + content.Datastore = accessInfo + + if srcHostCacheImage != nil { + err = srcHostCacheImage.AddDownloadRefcount() + if err != nil { + return err + } + srcHost, err := srcHostCacheImage.GetHost() + if err != nil { + return err + } + content.SrcHostIp = srcHost.AccessIp + content.SrcPath = srcHostCacheImage.Path + srcStorage := srcHost.GetStorageByFilePath(srcHostCacheImage.Path) + accessInfo, err := srcHost.GetCloudaccount().GetVCenterAccessInfo(srcStorage.ExternalId) + if err != nil { + return err + } + content.SrcDatastore = accessInfo + } + + agent, err := host.GetEsxiAgentHost() + if err != nil { + log.Errorf("find ESXi agent fail: %s", err) + return err + } + + if agent == nil { + return fmt.Errorf("fail to find valid ESXi agent") + } + + url := fmt.Sprintf("%s/disks/image_cache", agent.ManagerUri) + + if isForce { + content.IsForce = true + } + content.StoragecacheId = storageCache.Id + + body := jsonutils.NewDict() + body.Add(jsonutils.Marshal(&content), "disk") + + header := task.GetTaskRequestHeader() + + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + if err != nil { + return err + } + return nil +} + +func (self *SESXiHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { + agent, err := host.GetEsxiAgentHost() + if err != nil { + log.Errorf("find ESXi agent fail: %s", err) + return err + } + + if agent == nil { + return fmt.Errorf("fail to find valid ESXi agent") + } + + type specStruct struct { + Datastore models.SVCenterAccessInfo + HostIp string + Format string + } + + spec := specStruct{} + spec.HostIp = host.AccessIp + spec.Format = "vmdk" + + accessInfo, err := host.GetCloudaccount().GetVCenterAccessInfo(storage.ExternalId) + if err != nil { + return err + } + spec.Datastore = accessInfo + + body := jsonutils.NewDict() + body.Add(jsonutils.Marshal(&spec), "disk") + + url := fmt.Sprintf("/disks/agent/create/%s", disk.Id) + + header := task.GetTaskRequestHeader() + + _, err = agent.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SESXiHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error { + agent, err := host.GetEsxiAgentHost() + if err != nil { + log.Errorf("find ESXi agent fail: %s", err) + return err + } + + if agent == nil { + return fmt.Errorf("fail to find valid ESXi agent") + } + + guests := disk.GetGuests() + if len(guests) == 0 { + return fmt.Errorf("No VM associate with this disk") + } + + if len(guests) > 1 { + return fmt.Errorf("The disk is attached to multiple guests") + } + + guest := guests[0] + + if guest.HostId != host.Id { + return fmt.Errorf("The only guest is not on the host????") + } + + type specStruct struct { + Vm models.SVCenterAccessInfo + Disk models.SVCenterAccessInfo + HostIp string + ImageId string + } + + spec := specStruct{} + spec.HostIp = host.AccessIp + spec.ImageId = imageId + + account := host.GetCloudaccount() + accessInfo, err := account.GetVCenterAccessInfo(guest.ExternalId) + if err != nil { + return err + } + spec.Vm = accessInfo + + accessInfo, err = account.GetVCenterAccessInfo(disk.ExternalId) + if err != nil { + return err + } + spec.Disk = accessInfo + + body := jsonutils.NewDict() + body.Add(jsonutils.Marshal(&spec), "disk") + + url := fmt.Sprintf("/disks/agent/save-prepare/%s", disk.Id) + header := task.GetTaskRequestHeader() + + _, err = agent.Request(task.GetUserCred(), "POST", url, header, body) + return err +} + +func (self *SESXiHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error { + + imagePath, _ := data.GetString("backup") + if len(imagePath) == 0 { + return fmt.Errorf("missing parameter backup") + } + agentId, _ := data.GetString("agent_id") + if len(agentId) == 0 { + return fmt.Errorf("missing parameter agent_id") + } + + agent := models.HostManager.FetchHostById(agentId) + if agent == nil { + return fmt.Errorf("cannot find host with id %s", agentId) + } + + storage := disk.GetStorage() + + type specStruct struct { + ImagePath string + ImageId string + StorageId string + StoragecacheId string + Compress bool `json:",allowempty"` + } + + spec := specStruct{} + spec.ImageId = imageId + spec.ImagePath = imagePath + spec.StorageId = storage.Id + spec.StoragecacheId = storage.StoragecacheId + spec.Compress = false + + body := jsonutils.NewDict() + body.Add(jsonutils.Marshal(&spec), "disk") + + url := "/disks/agent/upload" + + header := task.GetTaskRequestHeader() + + _, err := agent.Request(task.GetUserCred(), "POST", url, header, body) + return err +} diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 93b5cab4b6..0eb2149043 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -3,7 +3,6 @@ package hostdrivers import ( "context" "fmt" - "net/http" "net/url" "yunion.io/x/jsonutils" @@ -17,7 +16,7 @@ import ( ) type SKVMHostDriver struct { - SBaseHostDriver + SVirtualizationHostDriver } func init() { @@ -46,8 +45,16 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod return err } - content := jsonutils.NewDict() - content.Add(jsonutils.NewString(imageId), "image_id") + type contentStruct struct { + ImageId string + SrcUrl string + IsForce bool + StoragecacheId string + } + + content := contentStruct{} + content.ImageId = imageId + if srcHostCacheImage != nil { err = srcHostCacheImage.AddDownloadRefcount() if err != nil { @@ -57,21 +64,19 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod if err != nil { return err } - srcUrl := fmt.Sprintf("%s/download/images/%s", srcHost.ManagerUri, imageId) - content.Add(jsonutils.NewString(srcUrl), "src_url") + content.SrcUrl = fmt.Sprintf("%s/download/images/%s", srcHost.ManagerUri, imageId) } url := fmt.Sprintf("%s/disks/image_cache", host.ManagerUri) if isForce { - content.Add(jsonutils.NewBool(true), "is_force") + content.IsForce = true } - content.Add(jsonutils.NewString(storageCache.Id), "storagecache_id") + content.StoragecacheId = storageCache.Id body := jsonutils.NewDict() - body.Add(content, "disk") - header := http.Header{} - header.Set("X-Auth-Token", task.GetUserCred().GetTokenString()) - header.Set("X-Task-Id", task.GetTaskId()) - header.Set("X-Region-Version", "v2") + body.Add(jsonutils.Marshal(&content), "disk") + + header := task.GetTaskRequestHeader() + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) if err != nil { return err @@ -79,10 +84,39 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mod return nil } +func (self *SKVMHostDriver) RequestUncacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { + type contentStruct struct { + ImageId string + StoragecacheId string + } + + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + + content := contentStruct{} + content.ImageId = imageId + content.StoragecacheId = storageCache.Id + + url := fmt.Sprintf("%s/disks/image_cache", host.ManagerUri) + + body := jsonutils.NewDict() + body.Add(jsonutils.Marshal(&content), "disk") + + header := task.GetTaskRequestHeader() + + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "DELETE", url, header, body, false) + if err != nil { + return err + } + return nil +} + func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") + header := task.GetTaskRequestHeader() + url := fmt.Sprintf("/disks/%s/create/%s", storage.Id, disk.Id) body := jsonutils.NewDict() body.Add(content, "disk") @@ -92,38 +126,36 @@ func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, ho func (self *SKVMHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { log.Infof("Deallocating disk on host %s", host.GetName()) - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") + header := task.GetTaskRequestHeader() + url := fmt.Sprintf("/disks/%s/delete/%s", storage.Id, disk.Id) body := jsonutils.NewDict() _, err := host.Request(task.GetUserCred(), "POST", url, header, body) return err } -func (self *SKVMHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") +func (self *SKVMHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, sizeMb int64, task taskman.ITask) error { + header := task.GetTaskRequestHeader() + url := fmt.Sprintf("/disks/%s/resize/%s", storage.Id, disk.Id) body := jsonutils.NewDict() content := jsonutils.NewDict() - content.Add(jsonutils.NewInt(size), "size") + content.Add(jsonutils.NewInt(sizeMb), "size") body.Add(content, "disk") _, err := host.Request(task.GetUserCred(), "POST", url, header, body) return err } -func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") +func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, sizeMb int64, task taskman.ITask) error { + self.RequestResizeDiskOnHost(ctx, host, storage, disk, sizeMb, task) + + header := task.GetTaskRequestHeader() + for _, guest := range disk.GetAttachedGuests() { guestdisk := guest.GetGuestDisk(disk.GetId()) url := fmt.Sprintf("/servers/%s/monitor", guest.GetId()) body := jsonutils.NewDict() - cmd := fmt.Sprintf("block_resize drive_%d %dM", guestdisk.Index, size) + cmd := fmt.Sprintf("block_resize drive_%d %dM", guestdisk.Index, sizeMb) body.Add(jsonutils.NewString(cmd), "cmd") host.Request(task.GetUserCred(), "POST", url, header, body) } @@ -134,7 +166,9 @@ func (self *SKVMHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, ho body := jsonutils.NewDict() body.Add(jsonutils.Marshal(map[string]string{"image_id": imageId}), "disk") url := fmt.Sprintf("/disks/%s/save-prepare/%s", disk.StorageId, disk.Id) - header := http.Header{"X-Task-Id": []string{task.GetTaskId()}, "X-Region-Version": []string{"v2"}} + + header := task.GetTaskRequestHeader() + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) return err } @@ -148,7 +182,9 @@ func (self *SKVMHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, ho } body.Add(jsonutils.Marshal(content), "disk") url := fmt.Sprintf("/disks/%s/upload", disk.StorageId) - header := http.Header{"X-Task-Id": []string{task.GetTaskId()}, "X-Region-Version": []string{"v2"}} + + header := task.GetTaskRequestHeader() + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) return err } @@ -157,27 +193,27 @@ func (self *SKVMHostDriver) RequestDeleteSnapshotsWithStorage(ctx context.Contex url := fmt.Sprintf("/storages/%s/delete-snapshots", snapshot.StorageId) body := jsonutils.NewDict() body.Set("disk_id", jsonutils.NewString(snapshot.DiskId)) - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") + + header := task.GetTaskRequestHeader() + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) return err } func (self *SKVMHostDriver) RequestResetDisk(ctx context.Context, host *models.SHost, disk *models.SDisk, params *jsonutils.JSONDict, task taskman.ITask) error { url := fmt.Sprintf("/disks/%s/reset/%s", disk.StorageId, disk.Id) - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") + + header := task.GetTaskRequestHeader() + _, err := host.Request(task.GetUserCred(), "POST", url, header, params) return err } func (self *SKVMHostDriver) RequestCleanUpDiskSnapshots(ctx context.Context, host *models.SHost, disk *models.SDisk, params *jsonutils.JSONDict, task taskman.ITask) error { url := fmt.Sprintf("/disks/%s/cleanup-snapshots/%s", disk.StorageId, disk.Id) - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") + + header := task.GetTaskRequestHeader() + _, err := host.Request(task.GetUserCred(), "POST", url, header, params) return err } diff --git a/pkg/compute/hostdrivers/managedvirtual.go b/pkg/compute/hostdrivers/managedvirtual.go new file mode 100644 index 0000000000..fa0bf40e7f --- /dev/null +++ b/pkg/compute/hostdrivers/managedvirtual.go @@ -0,0 +1,289 @@ +package hostdrivers + +import ( + "context" + "fmt" + "os" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/options" + + "yunion.io/x/onecloud/pkg/compute/models" +) + +type SManagedVirtualizationHostDriver struct { + SVirtualizationHostDriver +} + +func (self *SManagedVirtualizationHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + + osArch, _ := params.GetString("os_arch") + osType, _ := params.GetString("os_type") + osDist, _ := params.GetString("os_distribution") + + isForce := jsonutils.QueryBoolean(params, "is_force", false) + userCred := task.GetUserCred() + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + + lockman.LockRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) + defer lockman.ReleaseRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) + + scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), storageCache.Id, imageId) + + iStorageCache, err := storageCache.GetIStorageCache() + if err != nil { + return nil, err + } + + extImgId, err := iStorageCache.UploadImage(userCred, imageId, osArch, osType, osDist, scimg.ExternalId, isForce) + + if err != nil { + return nil, err + } + + // scimg.SetExternalId(extImgId) + + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(extImgId), "image_id") + return ret, nil + }) + return nil +} + +func (self *SManagedVirtualizationHostDriver) RequestUncacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { + params := task.GetParams() + imageId, err := params.GetString("image_id") + if err != nil { + return err + } + + scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), storageCache.Id, imageId) + if scimg == nil { + task.ScheduleRun(nil) + return nil + } + + if len(scimg.ExternalId) == 0 { + log.Errorf("cached image has not external ID???") + task.ScheduleRun(nil) + return nil + } + + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + lockman.LockRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) + defer lockman.ReleaseRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) + + iStorageCache, err := storageCache.GetIStorageCache() + if err != nil { + log.Errorf("GetIStorageCache fail %s", err) + return nil, err + } + + iImage, err := iStorageCache.GetIImageById(scimg.ExternalId) + if err != nil { + if err == cloudprovider.ErrNotFound { + return nil, nil + } + log.Errorf("GetIImageById fail %s", err) + return nil, err + } + + err = iImage.Delete(ctx) + if err != nil { + log.Errorf("iImage Delete fail %s", err) + return nil, err + } + + return nil, nil + }) + return nil +} + +func (self *SManagedVirtualizationHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} + +func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error { + iDisk, err := disk.GetIDisk() + if err != nil { + return err + } + iStorage, err := disk.GetIStorage() + if err != nil { + return err + } + iStoragecache := iStorage.GetIStoragecache() + if iStoragecache == nil { + return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) + } + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage") + if err != nil { + return nil, err + } + params := task.GetParams() + osType, _ := params.GetString("properties", "os_type") + + scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId) + if scimg.Status != models.CACHED_IMAGE_STATUS_READY { + scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host") + } + iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, "") + if err != nil { + log.Errorf("fail to create iImage: %v", err) + scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) + return nil, err + } + scimg.SetExternalId(iImage.GetId()) + if _, err := os.Stat(options.Options.TempPath); os.IsNotExist(err) { + if err = os.MkdirAll(options.Options.TempPath, 0755); err != nil { + return nil, err + } + } + result, err := iStoragecache.DownloadImage(task.GetUserCred(), imageId, iImage.GetId(), options.Options.TempPath) + if err != nil { + scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) + return nil, err + } + if err := iImage.Delete(ctx); err != nil { + log.Errorf("Delete iImage %s failed: %v", iImage.GetId(), err) + } + if err := snapshot.Delete(); err != nil { + log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err) + } + scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_READY, "") + return result, nil + }) + return nil +} + +func (self *SManagedVirtualizationHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { + return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) +} + +func (self *SManagedVirtualizationHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, sizeMb int64, task taskman.ITask) error { + iCloudStorage, err := storage.GetIStorage() + if err != nil { + log.Errorf("storage.GetIStorage fail %s", err) + return err + } + + iDisk, err := iCloudStorage.GetIDiskById(disk.GetExternalId()) + if err != nil { + log.Errorf("iCloudStorage.GetIDisk fail %s", err) + return err + } + + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + err = iDisk.Resize(ctx, sizeMb) + if err != nil { + log.Errorf("iDisk.Resize fail %s", err) + return nil, err + } + return jsonutils.Marshal(map[string]int64{"disk_size": sizeMb}), nil + }) + + return nil +} + +func (self *SManagedVirtualizationHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { + iCloudStorage, err := storage.GetIStorage() + if err != nil { + return err + } + size, err := content.Int("size") + if err != nil { + return err + } + size = size >> 10 + + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + iDisk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), "") + if err != nil { + return nil, err + } + _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.ExternalId = iDisk.GetGlobalId() + return nil + }) + if err != nil { + log.Errorf("Update disk externalId err: %v", err) + return nil, err + } + + if metaData := iDisk.GetMetadata(); metaData != nil { + meta := make(map[string]string) + if err := metaData.Unmarshal(meta); err != nil { + log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) + } else { + for key, value := range meta { + if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { + log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) + } + } + } + } + + data := jsonutils.NewDict() + data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") + data.Add(jsonutils.NewString(iDisk.GetDiskFormat()), "disk_format") + + return data, nil + }) + + return nil +} + +func (self *SManagedVirtualizationHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { + data := jsonutils.NewDict() + + iCloudStorage, err := storage.GetIStorage() + if err != nil { + return err + } + + iDisk, err := iCloudStorage.GetIDiskById(disk.GetExternalId()) + if err != nil { + if err == cloudprovider.ErrNotFound { + task.ScheduleRun(data) + return nil + } + return err + } + + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + err := iDisk.Delete(ctx) + return nil, err + }) + + return nil +} + +func (self *SAliyunHostDriver) RequestResetDisk(ctx context.Context, host *models.SHost, disk *models.SDisk, params *jsonutils.JSONDict, task taskman.ITask) error { + iDisk, err := disk.GetIDisk() + if err != nil { + return err + } + snapshotId, err := params.GetString("snapshot_id") + if err != nil { + return err + } + taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { + err := iDisk.Reset(ctx, snapshotId) + return nil, err + }) + return nil +} diff --git a/pkg/compute/hostdrivers/qcloud.go b/pkg/compute/hostdrivers/qcloud.go index 7a30ca306a..0cacf12199 100644 --- a/pkg/compute/hostdrivers/qcloud.go +++ b/pkg/compute/hostdrivers/qcloud.go @@ -2,21 +2,13 @@ package hostdrivers import ( "context" - "fmt" - "os" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" ) type SQcloudHostDriver struct { - SBaseHostDriver + SManagedVirtualizationHostDriver } func init() { @@ -28,182 +20,6 @@ func (self *SQcloudHostDriver) GetHostType() string { return models.HOST_TYPE_QCLOUD } -func (self *SQcloudHostDriver) CheckAndSetCacheImage(ctx context.Context, host *models.SHost, storageCache *models.SStoragecache, task taskman.ITask) error { - params := task.GetParams() - imageId, err := params.GetString("image_id") - if err != nil { - return err - } - - osArch, _ := params.GetString("os_arch") - osType, _ := params.GetString("os_type") - osDist, _ := params.GetString("os_distribution") - - isForce := jsonutils.QueryBoolean(params, "is_force", false) - userCred := task.GetUserCred() - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - lockman.LockRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - defer lockman.ReleaseRawObject(ctx, "cachedimages", fmt.Sprintf("%s-%s", storageCache.Id, imageId)) - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), storageCache.Id, imageId) - iStorageCache, err := storageCache.GetIStorageCache() - if err != nil { - return nil, err - } - - extImgId, err := iStorageCache.UploadImage(userCred, imageId, osArch, osType, osDist, scimg.ExternalId, isForce) - if err != nil { - return nil, err - } - scimg.SetExternalId(extImgId) - - ret := jsonutils.NewDict() - ret.Add(jsonutils.NewString(extImgId), "image_id") - return ret, nil - }) - return nil -} - -func (self *SQcloudHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { - iCloudStorage, err := storage.GetIStorage() - if err != nil { - return err - } - size, err := content.Int("size") - if err != nil { - return err - } - size = size >> 10 - iDisk, err := iCloudStorage.CreateIDisk(disk.GetName(), int(size), "") - if err != nil { - return err - } - _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { - disk.ExternalId = iDisk.GetGlobalId() - - if metaData := iDisk.GetMetadata(); metaData != nil { - meta := make(map[string]string) - if err := metaData.Unmarshal(meta); err != nil { - log.Errorf("Get disk %s Metadata error: %v", disk.Name, err) - } else { - for key, value := range meta { - if err := disk.SetMetadata(ctx, key, value, task.GetUserCred()); err != nil { - log.Errorf("set disk %s mata %s => %s error: %v", disk.Name, key, value, err) - } - } - } - } - - return nil - }) - if err != nil { - log.Errorf("Update disk externalId err: %v", err) - return err - } - data := jsonutils.NewDict() - data.Add(jsonutils.NewInt(int64(iDisk.GetDiskSizeMB())), "disk_size") - data.Add(jsonutils.NewString(iDisk.GetDiskFormat()), "disk_format") - task.ScheduleRun(data) - return nil -} - -func (self *SQcloudHostDriver) RequestDeallocateDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask) error { - data := jsonutils.NewDict() - if iCloudStorage, err := storage.GetIStorage(); err != nil { - return err - } else if iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()); err != nil { - if err == cloudprovider.ErrNotFound { - task.ScheduleRun(data) - return nil - } - return err - } else if err := iDisk.Delete(ctx); err != nil { - return err - } - task.ScheduleRun(data) - return nil -} - -func (self *SQcloudHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - return self.RequestResizeDiskOnHost(ctx, host, storage, disk, size, task) -} - -func (self *SQcloudHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, size int64, task taskman.ITask) error { - iCloudStorage, err := storage.GetIStorage() - if err != nil { - return err - } - iDisk, err := iCloudStorage.GetIDisk(disk.GetExternalId()) - if err != nil { - return err - } - err = iDisk.Resize(ctx, size>>10) - if err != nil { - return err - } - task.ScheduleRun(jsonutils.Marshal(map[string]int64{"disk_size": size})) - return nil -} - -func (self *SQcloudHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error { - task.ScheduleRun(nil) - return nil -} - -func (self *SQcloudHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error { - iDisk, err := disk.GetIDisk() - if err != nil { - return err - } - iStorage, err := disk.GetIStorage() - if err != nil { - return err - } - iStoragecache := iStorage.GetIStoragecache() - if iStoragecache == nil { - return httperrors.NewResourceNotFoundError("fail to find iStoragecache for storage: %s", iStorage.GetName()) - } - taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - snapshot, err := iDisk.CreateISnapshot(ctx, fmt.Sprintf("Snapshot-%s", imageId), "PrepareSaveImage") - if err != nil { - return nil, err - } - params := task.GetParams() - osType, _ := params.GetString("properties", "os_type") - - scimg := models.StoragecachedimageManager.Register(ctx, task.GetUserCred(), iStoragecache.GetId(), imageId) - if scimg.Status != models.CACHED_IMAGE_STATUS_READY { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHING, "request_prepare_save_disk_on_host") - } - iImage, err := iStoragecache.CreateIImage(snapshot.GetId(), fmt.Sprintf("Image-%s", imageId), osType, "") - if err != nil { - log.Errorf("fail to create iImage: %v", err) - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } - scimg.SetExternalId(iImage.GetId()) - if _, err := os.Stat(options.Options.TempPath); os.IsNotExist(err) { - if err = os.MkdirAll(options.Options.TempPath, 0755); err != nil { - return nil, err - } - } - result, err := iStoragecache.DownloadImage(task.GetUserCred(), imageId, iImage.GetId(), options.Options.TempPath) - if err != nil { - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - return nil, err - } - if err := iImage.Delete(); err != nil { - log.Errorf("Delete iImage %s failed: %v", iImage.GetId(), err) - } - if err := snapshot.Delete(); err != nil { - log.Errorf("Delete snapshot %s failed: %v", snapshot.GetId(), err) - } - scimg.SetStatus(task.GetUserCred(), models.CACHED_IMAGE_STATUS_READY, "") - return result, nil - }) - return nil -} - func (self *SQcloudHostDriver) RequestDeleteSnapshotWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error { return httperrors.NewNotImplementedError("not implement") } diff --git a/pkg/compute/hostdrivers/virtualization.go b/pkg/compute/hostdrivers/virtualization.go new file mode 100644 index 0000000000..d91c5105a5 --- /dev/null +++ b/pkg/compute/hostdrivers/virtualization.go @@ -0,0 +1,5 @@ +package hostdrivers + +type SVirtualizationHostDriver struct { + SBaseHostDriver +} diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index 3d49d59128..4093d90ba2 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -60,7 +60,7 @@ func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error { if self.getStoragecacheCount() > 0 { return httperrors.NewNotEmptyError("The image has been cached on storages") } - if self.getStatus() == "active" && !self.isReferenceSessionExpire() { + if self.GetStatus() == "active" && !self.isReferenceSessionExpire() { return httperrors.NewConflictError("the image reference session has not been expired!") } return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx) @@ -82,27 +82,27 @@ func (self *SCachedimage) isRefreshSessionExpire() bool { } } -func (self *SCachedimage) getName() string { +func (self *SCachedimage) GetName() string { name, _ := self.Info.GetString("name") return name } -func (self *SCachedimage) getOwner() string { +func (self *SCachedimage) GetOwner() string { owner, _ := self.Info.GetString("owner") return owner } -func (self *SCachedimage) getFormat() string { +func (self *SCachedimage) GetFormat() string { format, _ := self.Info.GetString("disk_format") return format } -func (self *SCachedimage) getStatus() string { +func (self *SCachedimage) GetStatus() string { status, _ := self.Info.GetString("status") return status } -func (self *SCachedimage) getOSType() string { +func (self *SCachedimage) GetOSType() string { osType, _ := self.Info.GetString("properties", "os_type") return osType } @@ -116,7 +116,7 @@ func (self *SCachedimage) getStoragecacheCount() int { return self.getStoragecacheQuery().Count() } -func (self *SCachedimage) getImage() (*SImage, error) { +func (self *SCachedimage) GetImage() (*SImage, error) { image := SImage{} err := self.Info.Unmarshal(&image) @@ -183,8 +183,8 @@ func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred m imgObj, _ := manager.FetchById(imageId) if imgObj != nil { cachedImage := imgObj.(*SCachedimage) - if cachedImage.getStatus() == "active" && len(cachedImage.getOSType()) > 0 && cachedImage.isRefreshSessionExpire() { - return cachedImage.getImage() + if cachedImage.GetStatus() == "active" && len(cachedImage.GetOSType()) > 0 && cachedImage.isRefreshSessionExpire() { + return cachedImage.GetImage() } } } @@ -198,7 +198,7 @@ func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred m if err != nil { return nil, err } - return cachedImage.getImage() + return cachedImage.GetImage() } func (manager *SCachedimageManager) getImageByName(ctx context.Context, userCred mcclient.TokenCredential, imageId string) (*SImage, error) { @@ -211,7 +211,7 @@ func (manager *SCachedimageManager) getImageByName(ctx context.Context, userCred if err != nil { return nil, err } - return cachedImage.getImage() + return cachedImage.GetImage() } func (manager *SCachedimageManager) getImageInfo(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { @@ -225,10 +225,10 @@ func (manager *SCachedimageManager) getImageInfo(ctx context.Context, userCred m func (self *SCachedimage) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := self.SStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query) - extra.Add(jsonutils.NewString(self.getName()), "name") - extra.Add(jsonutils.NewString(self.getOwner()), "owner") - extra.Add(jsonutils.NewString(self.getFormat()), "format") - extra.Add(jsonutils.NewString(self.getStatus()), "status") + extra.Add(jsonutils.NewString(self.GetName()), "name") + extra.Add(jsonutils.NewString(self.GetOwner()), "owner") + extra.Add(jsonutils.NewString(self.GetFormat()), "format") + extra.Add(jsonutils.NewString(self.GetStatus()), "status") for _, k := range []string{"os_type", "os_distribution", "os_version", "hypervisor"} { val, _ := self.Info.GetString("properties", k) if len(val) > 0 { @@ -252,7 +252,7 @@ func (self *SCachedimage) PerformRefresh(ctx context.Context, userCred mcclient. } func (self *SCachedimage) addRefCount() { - if self.getStatus() != "active" { + if self.GetStatus() != "active" { return } _, err := CachedimageManager.TableSpec().Update(self, func() error { @@ -294,7 +294,7 @@ func (self *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, exclu for _, obj := range v { q = q.Filter(sqlchemy.Equals(host.Field("zone_id"), obj.Id)) } - case []*SVCenter: + case []*SCloudprovider: for _, obj := range v { q = q.Filter(sqlchemy.Equals(host.Field("manager_id"), obj.Id)) } diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index f3f65d5784..34e6949406 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -13,6 +13,8 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "net/url" + "strconv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -478,7 +480,9 @@ func (manager *SCloudaccountManager) FetchCloudaccountByIdOrName(accountId strin } func (self *SCloudaccount) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { - extra.Add(jsonutils.Marshal(self.GetCloudproviders()), "accounts") + providers := self.GetCloudproviders() + extra.Add(jsonutils.NewInt(int64(len(providers))), "account_count") + extra.Add(jsonutils.Marshal(providers), "accounts") return extra } @@ -602,3 +606,54 @@ func (self *SCloudaccount) GetDetailsBalance(ctx context.Context, userCred mccli ret.Add(jsonutils.NewFloat(balance), "balance") return ret, nil } + +func (self *SCloudaccount) getHostPort() (string, int, error) { + urlComponent, err := url.Parse(self.AccessUrl) + if err != nil { + return "", 0, err + } + host := urlComponent.Hostname() + portStr := urlComponent.Port() + port := 0 + if len(portStr) > 0 { + port, err = strconv.Atoi(portStr) + if err != nil { + return "", 0, err + } + } + if port == 0 { + if urlComponent.Scheme == "http" { + port = 80 + } else if urlComponent.Scheme == "https" { + port = 443 + } + } + return host, port, nil +} + +type SVCenterAccessInfo struct { + VcenterId string + Host string + Port int + Account string + Password string + PrivateId string +} + +func (self *SCloudaccount) GetVCenterAccessInfo(privateId string) (SVCenterAccessInfo, error) { + info := SVCenterAccessInfo{} + + host, port, err := self.getHostPort() + if err != nil { + return info, err + } + + info.VcenterId = self.Id + info.Host = host + info.Port = port + info.Account = self.Account + info.Password = self.Secret + info.PrivateId = privateId + + return info, nil +} diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index c5bf013aca..c1125abc29 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -600,22 +600,24 @@ func (self *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCre } func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { - if storage := self.GetStorage(); storage == nil { + storage := self.GetStorage() + if storage == nil { return nil, httperrors.NewResourceNotFoundError("fail to find storage for disk %s", self.GetName()) - } else if provider, err := storage.GetDriver(); err != nil { - return nil, err - } else { - return provider.GetIStorageById(storage.GetExternalId()) } + istorage, err := storage.GetIStorage() + if err != nil { + return nil, err + } + return istorage, nil } func (self *SDisk) GetIDisk() (cloudprovider.ICloudDisk, error) { - if iStorage, err := self.GetIStorage(); err != nil { + iStorage, err := self.GetIStorage() + if err != nil { log.Errorf("fail to find iStorage: %v", err) return nil, err - } else { - return iStorage.GetIDisk(self.GetExternalId()) } + return iStorage.GetIDiskById(self.GetExternalId()) } func (self *SDisk) GetZone() *SZone { @@ -839,6 +841,7 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke self.Status = extDisk.GetStatus() self.DiskFormat = extDisk.GetDiskFormat() self.DiskSize = extDisk.GetDiskSizeMB() + self.AccessPath = extDisk.GetAccessPath() if extDisk.GetIsAutoDelete() { self.AutoDelete = true } @@ -1199,9 +1202,9 @@ func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.To return self.getMoreDetails(extra) } -func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, size int64, parentTaskId string, pendingUsage quotas.IQuota) error { +func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, sizeMb int64, parentTaskId string, pendingUsage quotas.IQuota) error { params := jsonutils.NewDict() - params.Add(jsonutils.NewInt(size), "size") + params.Add(jsonutils.NewInt(sizeMb), "size") if task, err := taskman.TaskManager.NewTask(ctx, "DiskResizeTask", self, userCred, params, parentTaskId, "", pendingUsage); err != nil { return err } else { diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index cb5834f328..becf1938aa 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -202,7 +202,7 @@ func (self *SGuestdisk) GetDetailedJson() *jsonutils.JSONDict { cachedImageObj, _ := CachedimageManager.FetchById(imageId) if cachedImageObj != nil { cachedImage := cachedImageObj.(*SCachedimage) - desc.Add(jsonutils.NewString(cachedImage.getName()), "image") + desc.Add(jsonutils.NewString(cachedImage.GetName()), "image") } } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index fef2cebfc6..246b9405ca 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3199,6 +3199,9 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T return nil, httperrors.NewBadRequestError("Params vcpu_count parse error") } addCpu = int(nVcpu - int64(self.VcpuCount)) + if addCpu < 0 { + addCpu = 0 + } } vmemSize, err := data.GetString("vmem_size") if err == nil { @@ -3214,6 +3217,9 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T return nil, httperrors.NewBadRequestError("Params vmem_size parse error") } addMem = nVmem - self.VmemSize + if addMem < 0 { + addMem = 0 + } } disks := self.GetDisks() var addDisk int diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go index 606f730639..2e490fc365 100644 --- a/pkg/compute/models/hostdrivers.go +++ b/pkg/compute/models/hostdrivers.go @@ -12,7 +12,10 @@ import ( type IHostDriver interface { GetHostType() string + CheckAndSetCacheImage(ctx context.Context, host *SHost, storagecache *SStoragecache, task taskman.ITask) error + RequestUncacheImage(ctx context.Context, host *SHost, storageCache *SStoragecache, task taskman.ITask) error + ValidateUpdateDisk(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) RequestPrepareSaveDiskOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask) error RequestSaveUploadImageOnHost(ctx context.Context, host *SHost, disk *SDisk, imageId string, task taskman.ITask, data jsonutils.JSONObject) error diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 880e064ca1..5371c46890 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -2,6 +2,7 @@ package models import ( "context" + "database/sql" "fmt" "net/http" "net/url" @@ -10,6 +11,8 @@ import ( "strings" "time" + "github.com/serialx/hashring" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/tristate" @@ -21,7 +24,6 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" - "database/sql" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -459,6 +461,19 @@ func (self *SHost) GetHoststorageOfId(storageId string) *SHoststorage { return &hoststorage } +func (self *SHost) GetStorageByFilePath(path string) *SStorage { + hoststorages := self.GetHoststorages() + if hoststorages == nil { + return nil + } + for i := 0; i < len(hoststorages); i += 1 { + if strings.HasPrefix(path, hoststorages[i].MountPoint) { + return hoststorages[i].GetStorage() + } + } + return nil +} + func (self *SHost) GetBaremetalstorage() *SHoststorage { if !self.IsBaremetal { return nil @@ -1125,7 +1140,9 @@ func (manager *SHostManager) newFromCloudHost(extHost cloudprovider.ICloudHost, return &host, nil } -func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, storages []cloudprovider.ICloudStorage) compare.SyncResult { +func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, storages []cloudprovider.ICloudStorage) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { + localStorages := make([]SStorage, 0) + remoteStorages := make([]cloudprovider.ICloudStorage, 0) syncResult := compare.SyncResult{} dbStorages := make([]SStorage, 0) @@ -1150,7 +1167,7 @@ func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.Token err := compare.CompareSets(dbStorages, storages, &removed, &commondb, &commonext, &added) if err != nil { syncResult.Error(err) - return syncResult + return nil, nil, syncResult } for i := 0; i < len(removed); i += 1 { @@ -1166,29 +1183,34 @@ func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.Token for i := 0; i < len(commondb); i += 1 { log.Infof("host %s is still connected with %s, to update ...", self.Id, commondb[i].Id) - err := self.syncWithCloudHostStorage(commonext[i]) + err := self.syncWithCloudHostStorage(&commondb[i], commonext[i]) if err != nil { syncResult.UpdateError(err) } else { syncResult.Update() } + localStorages = append(localStorages, commondb[i]) + remoteStorages = append(remoteStorages, commonext[i]) } for i := 0; i < len(added); i += 1 { log.Infof("host %s is found connected with %s, to add ...", self.Id, added[i].GetId()) - err := self.newCloudHostStorage(ctx, userCred, added[i]) + local, err := self.newCloudHostStorage(ctx, userCred, added[i]) if err != nil { syncResult.AddError(err) } else { syncResult.Add() } + localStorages = append(localStorages, *local) + remoteStorages = append(remoteStorages, added[i]) } - return syncResult + return localStorages, remoteStorages, syncResult } -func (self *SHost) syncWithCloudHostStorage(extStorage cloudprovider.ICloudStorage) error { +func (self *SHost) syncWithCloudHostStorage(localStorage *SStorage, extStorage cloudprovider.ICloudStorage) error { // do nothing - return nil + hs := self.GetHoststorageOfId(localStorage.Id) + return hs.syncWithCloudHostStorage(extStorage) } func (self *SHost) Attach2Storage(ctx context.Context, userCred mcclient.TokenCredential, storage *SStorage, mountPoint string) error { @@ -1207,25 +1229,25 @@ func (self *SHost) Attach2Storage(ctx context.Context, userCred mcclient.TokenCr return nil } -func (self *SHost) newCloudHostStorage(ctx context.Context, userCred mcclient.TokenCredential, extStorage cloudprovider.ICloudStorage) error { +func (self *SHost) newCloudHostStorage(ctx context.Context, userCred mcclient.TokenCredential, extStorage cloudprovider.ICloudStorage) (*SStorage, error) { storageObj, err := StorageManager.FetchByExternalId(extStorage.GetGlobalId()) if err != nil { if err == sql.ErrNoRows { // no cloud storage found, this may happen for on-premise host - // create the storage + // create the storage right now storageObj, err = StorageManager.newFromCloudStorage(extStorage, self.GetZone()) if err != nil { log.Errorf("create by cloud storage fail %s", err) - return err + return nil, err } } else { log.Errorf("%s", err) - return err + return nil, err } } storage := storageObj.(*SStorage) - err = self.Attach2Storage(ctx, userCred, storage, "") - return err + err = self.Attach2Storage(ctx, userCred, storage, extStorage.GetMountPoint()) + return storage, err } func (self *SHost) SyncHostWires(ctx context.Context, userCred mcclient.TokenCredential, wires []cloudprovider.ICloudWire) compare.SyncResult { @@ -1656,7 +1678,23 @@ func (self *SHost) GetIHost() (cloudprovider.ICloudHost, error) { if err != nil { return nil, fmt.Errorf("No cloudprovide for host: %s", err) } - ihost, err := provider.GetIHostById(self.ExternalId) + var iregion cloudprovider.ICloudRegion + if provider.IsOnPremiseInfrastructure() { + iregion, err = provider.GetOnPremiseIRegion() + } else { + region := self.GetRegion() + if region == nil { + msg := "fail to find region of host???" + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + iregion, err = provider.GetIRegionById(region.ExternalId) + } + if err != nil { + log.Errorf("fail to find iregion: %s", err) + return nil, err + } + ihost, err := iregion.GetIHostById(self.ExternalId) if err != nil { log.Errorf("fail to find ihost by id %s %s", self.ExternalId, err) return nil, fmt.Errorf("fail to find ihost by id %s", err) @@ -3122,3 +3160,57 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T return result } + +func (manager *SHostManager) GetEsxiAgentHostId(key string) (string, error) { + q := HostManager.Query("id") + q = q.Equals("host_status", HOST_ONLINE) + q = q.Equals("host_type", HOST_TYPE_HYPERVISOR) + q = q.IsTrue("enabled") + + rows, err := q.Rows() + if err != nil { + return "", err + } + + var hostId string + hostIds := make([]string, 0) + for rows.Next() { + err = rows.Scan(&hostId) + if err != nil { + return "", err + } + hostIds = append(hostIds, hostId) + } + + ring := hashring.New(hostIds) + ret, _ := ring.GetNode(key) + return ret, nil +} + +func (manager *SHostManager) GetEsxiAgentHost(key string) (*SHost, error) { + hostId, err := manager.GetEsxiAgentHostId(key) + if err != nil { + return nil, err + } + return manager.FetchHostById(hostId), nil +} + +func (host *SHost) GetEsxiAgentHost() (*SHost, error) { + return HostManager.GetEsxiAgentHost(host.Id) +} + +func (manager *SHostManager) GetHostByIp(hostIp string) (*SHost, error) { + q := manager.Query() + q = q.Equals("access_ip", hostIp) + + host, err := db.NewModelObject(manager) + if err != nil { + return nil, err + } + err = q.First(host) + if err != nil { + return nil, err + } + + return host.(*SHost), nil +} diff --git a/pkg/compute/models/hoststorages.go b/pkg/compute/models/hoststorages.go index 8da2193ddb..a6d982719d 100644 --- a/pkg/compute/models/hoststorages.go +++ b/pkg/compute/models/hoststorages.go @@ -5,11 +5,13 @@ import ( "fmt" "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/tristate" "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SHoststorageManager struct { @@ -159,3 +161,11 @@ func (manager *SHoststorageManager) GetStorages(hostId string) ([]SHoststorage, } return hoststorage, nil } + +func (self *SHoststorage) syncWithCloudHostStorage(extStorage cloudprovider.ICloudStorage) error { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.MountPoint = extStorage.GetMountPoint() + return nil + }) + return err +} diff --git a/pkg/compute/models/managedresource.go b/pkg/compute/models/managedresource.go index fc6c7a0153..3b490b17da 100644 --- a/pkg/compute/models/managedresource.go +++ b/pkg/compute/models/managedresource.go @@ -19,6 +19,14 @@ func (self *SManagedResourceBase) GetCloudprovider() *SCloudprovider { return nil } +func (self *SManagedResourceBase) GetCloudaccount() *SCloudaccount { + cp := self.GetCloudprovider() + if cp == nil { + return nil + } + return cp.GetCloudaccount() +} + func (self *SManagedResourceBase) GetDriver() (cloudprovider.ICloudProvider, error) { provider := self.GetCloudprovider() if provider == nil { diff --git a/pkg/compute/models/storagecachedimages.go b/pkg/compute/models/storagecachedimages.go index 4a646c146d..a91cc895b1 100644 --- a/pkg/compute/models/storagecachedimages.go +++ b/pkg/compute/models/storagecachedimages.go @@ -153,7 +153,7 @@ func (self *SStoragecachedimage) getExtraDetails(extra *jsonutils.JSONDict) *jso } cachedImage := self.GetCachedimage() if cachedImage != nil { - extra.Add(jsonutils.NewString(cachedImage.getName()), "image") + extra.Add(jsonutils.NewString(cachedImage.GetName()), "image") extra.Add(jsonutils.NewInt(cachedImage.Size), "size") } extra.Add(jsonutils.NewInt(int64(self.getReferenceCount())), "reference") @@ -237,7 +237,8 @@ func (self *SStoragecachedimage) markDeleting(ctx context.Context, userCred mccl lockman.LockJointObject(ctx, cache, image) defer lockman.ReleaseJointObject(ctx, cache, image) - if !isForce && !utils.IsInStringArray(self.Status, []string{CACHED_IMAGE_STATUS_READY, CACHED_IMAGE_STATUS_DELETING}) { + if !isForce && !utils.IsInStringArray(self.Status, + []string{CACHED_IMAGE_STATUS_READY, CACHED_IMAGE_STATUS_DELETING, CACHED_IMAGE_STATUS_CACHE_FAILED}) { return httperrors.NewInvalidStatusError("Cannot uncache in status %s", self.Status) } _, err = self.GetModelManager().TableSpec().Update(self, func() error { diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index 240a99782b..c34fb61d9f 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/log" "yunion.io/x/sqlchemy" + "fmt" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -40,7 +41,7 @@ type SStoragecache struct { SInfrastructure SManagedResourceBase - Path string `width:"256" charset:"utf8" nullable:"true" list:"admin" update:"admin" create:"admin_optional"` // = Column(VARCHAR(256, charset='utf8'), nullable=True) + Path string `width:"256" charset:"utf8" nullable:"true" list:"user" update:"admin" create:"admin_optional"` // = Column(VARCHAR(256, charset='utf8'), nullable=True) } func (self *SStoragecache) getStorages() []SStorage { @@ -141,6 +142,8 @@ func (manager *SStoragecacheManager) newFromCloudStoragecache(cloudCache cloudpr local.IsEmulated = cloudCache.IsEmulated() local.ManagerId = cloudCache.GetManagerId() + local.Path = cloudCache.GetPath() + err := manager.TableSpec().Insert(&local) if err != nil { return nil, err @@ -153,6 +156,8 @@ func (self *SStoragecache) syncWithCloudStoragecache(cloudCache cloudprovider.IC _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.Name = cloudCache.GetName() + self.Path = cloudCache.GetPath() + self.IsEmulated = cloudCache.IsEmulated() self.ManagerId = cloudCache.GetManagerId() @@ -197,7 +202,9 @@ func (self *SStoragecache) getCachedImageSize() int64 { var size int64 = 0 for _, img := range images { imginfo := img.GetCachedimage() - size += imginfo.Size + if imginfo != nil { + size += imginfo.Size + } } return size } @@ -237,13 +244,6 @@ func (self *SStoragecache) StartImageCacheTask(ctx context.Context, userCred mcc } func (self *SStoragecache) StartImageUncacheTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, isForce bool, parentTaskId string) error { - if !isForce { - err := self.ValidateDeleteCondition(ctx) - if err != nil { - return err - } - } - data := jsonutils.NewDict() data.Add(jsonutils.NewString(imageId), "image_id") if isForce { @@ -258,12 +258,18 @@ func (self *SStoragecache) StartImageUncacheTask(ctx context.Context, userCred m } func (self *SStoragecache) GetIStorageCache() (cloudprovider.ICloudStoragecache, error) { - provider, err := self.GetDriver() + storages := self.getStorages() + if len(storages) == 0 { + msg := "no storages for this storagecache???" + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + istorage, err := storages[0].GetIStorage() if err != nil { - log.Errorf("fail to find cloud provider") + log.Errorf("fail to find istorage for storage %s", err) return nil, err } - return provider.GetIStoragecacheById(self.GetExternalId()) + return istorage.GetIStoragecache(), nil } func (manager *SStoragecacheManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index b7be546d7d..b4261c63d7 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -3,6 +3,7 @@ package models import ( "context" + "fmt" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -258,6 +259,14 @@ func (self *SStorage) getZone() *SZone { return nil } +func (self *SStorage) GetRegion() *SCloudregion { + zone := self.getZone() + if zone == nil { + return nil + } + return zone.GetRegion() +} + func (self *SStorage) GetReserved() int { return self.Reserved } @@ -705,7 +714,28 @@ func (self *SStorage) GetIStorage() (cloudprovider.ICloudStorage, error) { log.Errorf("fail to find cloud provider") return nil, err } - return provider.GetIStorageById(self.GetExternalId()) + var iRegion cloudprovider.ICloudRegion + if provider.IsOnPremiseInfrastructure() { + iRegion, err = provider.GetOnPremiseIRegion() + } else { + region := self.GetRegion() + if region == nil { + msg := "cannot find region for storage???" + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + iRegion, err = provider.GetIRegionById(region.ExternalId) + } + if err != nil { + log.Errorf("provider.GetIRegionById fail %s", err) + return nil, err + } + istore, err := iRegion.GetIStorageById(self.GetExternalId()) + if err != nil { + log.Errorf("iRegion.GetIStorageById fail %s", err) + return nil, err + } + return istore, nil } func (manager *SStorageManager) FetchStorageById(storageId string) *SStorage { @@ -807,3 +837,20 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. return q, err } + +func (self *SStorage) ClearSchedDescCache() error { + hosts := self.GetAllAttachingHosts() + if hosts == nil { + msg := "get attaching host error" + log.Errorf(msg) + return fmt.Errorf(msg) + } + for i := 0; i < len(hosts); i += 1 { + err := hosts[i].ClearSchedDescCache() + if err != nil { + log.Errorf("host CleanHostSchedCache error: %v", err) + return err + } + } + return nil +} diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index 20b3eaad36..19672f747d 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -411,7 +411,28 @@ func (self *SVpc) GetIVpc() (cloudprovider.ICloudVpc, error) { log.Errorf("fail to find cloud provider") return nil, err } - return provider.GetIVpcById(self.GetExternalId()) + var iregion cloudprovider.ICloudRegion + if provider.IsOnPremiseInfrastructure() { + iregion, err = provider.GetOnPremiseIRegion() + } else { + region := self.GetRegion() + if region == nil { + msg := "fail to find region of host???" + log.Errorf(msg) + return nil, fmt.Errorf(msg) + } + iregion, err = provider.GetIRegionById(region.ExternalId) + } + if err != nil { + log.Errorf("fail to find iregion: %s", err) + return nil, err + } + ivpc, err := iregion.GetIVpcById(self.ExternalId) + if err != nil { + log.Errorf("fail to find ivpc by id %s %s", self.ExternalId, err) + return nil, fmt.Errorf("fail to find ivpc by id %s", err) + } + return ivpc, nil } func (self *SVpc) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { diff --git a/pkg/compute/tasks/cloud_provider_sync_info_task.go b/pkg/compute/tasks/cloud_provider_sync_info_task.go index b12942501b..e24d743669 100644 --- a/pkg/compute/tasks/cloud_provider_sync_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_info_task.go @@ -381,7 +381,7 @@ func syncHostStorages(ctx context.Context, provider *models.SCloudprovider, task logSyncFailed(provider, task, msg) return } - result := localHost.SyncHostStorages(ctx, task.UserCred, storages) + localStorages, remoteStorages, result := localHost.SyncHostStorages(ctx, task.UserCred, storages) msg := result.Result() notes := fmt.Sprintf("SyncHostStorages for host %s result: %s", localHost.Name, msg) log.Infof(notes) @@ -391,6 +391,10 @@ func syncHostStorages(ctx context.Context, provider *models.SCloudprovider, task } db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, task.UserCred) logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) + + for i := 0; i < len(localStorages); i += 1 { + syncStorageCaches(ctx, provider, task, &localStorages[i], remoteStorages[i]) + } } func syncHostWires(ctx context.Context, provider *models.SCloudprovider, task taskman.ITask, localHost *models.SHost, remoteHost cloudprovider.ICloudHost) { diff --git a/pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go b/pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go index b0f07904ed..ceb15a3f7c 100644 --- a/pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go +++ b/pkg/compute/tasks/cloud_provider_sync_onpremise_info_task.go @@ -14,15 +14,23 @@ import ( ) func syncOnPremiseCloudProviderInfo(ctx context.Context, provider *models.SCloudprovider, task *CloudProviderSyncInfoTask, driver cloudprovider.ICloudProvider, syncRange *models.SSyncRange) { - ihosts, err := driver.GetOnPremiseIHosts() + iregion, err := driver.GetOnPremiseIRegion() if err != nil { - msg := fmt.Sprintf("GetOnPremiseIHosts for provider %s failed %s", provider.GetName(), err) + msg := fmt.Sprintf("GetOnPremiseIRegion for provider %s failed %s", provider.GetName(), err) log.Errorf(msg) logSyncFailed(provider, task, msg) return } - localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, nil, ihosts) + ihosts, err := iregion.GetIHosts() + if err != nil { + msg := fmt.Sprintf("GetIHosts for provider %s failed %s", provider.GetName(), err) + log.Errorf(msg) + logSyncFailed(provider, task, msg) + return + } + + localHosts, remoteHosts, result := models.HostManager.SyncHosts(ctx, task.UserCred, provider, nil, ihosts, syncRange.ProjectSync) msg := result.Result() notes := fmt.Sprintf("SyncHosts for provider %s result: %s", provider.Name, msg) log.Infof(notes) diff --git a/pkg/compute/tasks/disk_base_task.go b/pkg/compute/tasks/disk_base_task.go index 083f78d85d..29133ab173 100644 --- a/pkg/compute/tasks/disk_base_task.go +++ b/pkg/compute/tasks/disk_base_task.go @@ -3,8 +3,6 @@ package tasks import ( "context" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -35,14 +33,5 @@ func (self *SDiskBaseTask) finalReleasePendingUsage(ctx context.Context) { } func (self *SDiskBaseTask) CleanHostSchedCache(disk *models.SDisk) { - storage := disk.GetStorage() - if hosts := storage.GetAllAttachingHosts(); hosts == nil { - log.Errorf("get attaching host error") - } else { - for _, h := range hosts { - if err := h.ClearSchedDescCache(); err != nil { - log.Errorf("host CleanHostSchedCache error: %v", err) - } - } - } + disk.GetStorage().ClearSchedDescCache() } diff --git a/pkg/compute/tasks/disk_resize_task.go b/pkg/compute/tasks/disk_resize_task.go index 9724a171d8..443caf9bdd 100644 --- a/pkg/compute/tasks/disk_resize_task.go +++ b/pkg/compute/tasks/disk_resize_task.go @@ -49,12 +49,12 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, func (self *DiskResizeTask) StartResizeDisk(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, online bool) { log.Infof("Resizing disk on host %s ...", host.GetName()) self.SetStage("on_disk_resize_complete", nil) - size, _ := self.GetParams().Int("size") + sizeMb, _ := self.GetParams().Int("size") proc := host.GetHostDriver().RequestResizeDiskOnHost if online { proc = host.GetHostDriver().RequestResizeDiskOnHostOnline } - if err := proc(ctx, host, storage, disk, size, self); err != nil { + if err := proc(ctx, host, storage, disk, sizeMb, self); err != nil { log.Errorf("request_resize_disk_on_host: %v", err) self.OnStartResizeDiskFailed(ctx, disk, err) return @@ -80,7 +80,7 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode self.OnStartResizeDiskFailed(ctx, disk, err) return } - size, err := jSize.Int() + sizeMb, err := jSize.Int() if err != nil { log.Errorf("OnDiskResizeComplete error: %s", err.Error()) self.OnStartResizeDiskFailed(ctx, disk, err) @@ -89,7 +89,7 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode oldStatus := disk.Status _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { disk.Status = models.DISK_READY - disk.DiskSize = int(size) + disk.DiskSize = int(sizeMb) return nil }) if err != nil { @@ -107,8 +107,8 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode self.finalReleasePendingUsage(ctx) } -func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, reason error) { - disk.SetDiskReady(ctx, self.GetUserCred(), reason.Error()) +func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, reason jsonutils.JSONObject) { + disk.SetDiskReady(ctx, self.GetUserCred(), reason.String()) db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(), self.UserCred) - logclient.AddActionLog(disk, logclient.ACT_RESIZE, reason.Error(), self.UserCred, false) + logclient.AddActionLog(disk, logclient.ACT_RESIZE, reason.String(), self.UserCred, false) } diff --git a/pkg/compute/tasks/guest_change_config_task.go b/pkg/compute/tasks/guest_change_config_task.go index 6c1a9ea3da..4586541dcb 100644 --- a/pkg/compute/tasks/guest_change_config_task.go +++ b/pkg/compute/tasks/guest_change_config_task.go @@ -2,6 +2,7 @@ package tasks import ( "context" + "fmt" "yunion.io/x/jsonutils" @@ -32,69 +33,63 @@ func (self *GuestChangeConfigTask) OnInit(ctx context.Context, obj db.IStandalon } func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + iResizeDisks, err := self.Params.Get("resize") if iResizeDisks == nil || err != nil { - self.markStageFailed(obj, ctx, err.Error()) + self.markStageFailed(ctx, guest, err.Error()) return } resizeDisks := iResizeDisks.(*jsonutils.JSONArray) for i := 0; i < resizeDisks.Length(); i++ { iResizeSet, err := resizeDisks.GetAt(i) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("resizeDisks.GetAt fail %s", err)) return } resizeSet := iResizeSet.(*jsonutils.JSONArray) diskId, err := resizeSet.GetAt(0) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("resizeSet.GetAt(0) fail %s", err)) return } idStr, err := diskId.GetString() if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("diskId.GetString fail %s", err)) return } jSize, err := resizeSet.GetAt(1) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("resizeSet.GetAt(1) fail %s", err)) return } size, err := jSize.Int() if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("jSize.Int fail %s", err)) return } iDisk, err := models.DiskManager.FetchById(idStr) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("models.DiskManager.FetchById(idStr) fail %s", err)) return } disk := iDisk.(*models.SDisk) - if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(disk, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) - return - } if disk.DiskSize < int(size) { var pendingUsage models.SQuota err = self.GetPendingUsage(&pendingUsage) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(disk, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("self.GetPendingUsage(&pendingUsage) fail %s", err)) + return + } + err = disk.StartDiskResizeTask(ctx, self.UserCred, size, self.GetTaskId(), &pendingUsage) + if err != nil { + self.markStageFailed(ctx, guest, fmt.Sprintf("disk.StartDiskResizeTask fail %s", err)) return } - disk.StartDiskResizeTask(ctx, self.UserCred, size, self.GetTaskId(), &pendingUsage) return } } - guest := obj.(*models.SGuest) + self.DoCreateDisksTask(ctx, guest) } @@ -107,12 +102,11 @@ func (self *GuestChangeConfigTask) DoCreateDisksTask(ctx context.Context, guest data := (iCreateData).(*jsonutils.JSONDict) self.SetStage("on_create_disks_complete", nil) guest.StartGuestCreateDiskTask(ctx, self.UserCred, data, self.GetTaskId()) - } func (self *GuestChangeConfigTask) OnCreateDisksCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { - self.markStageFailed(obj, ctx, err.String()) - logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + guest := obj.(*models.SGuest) + self.markStageFailed(ctx, guest, fmt.Sprintf("OnCreateDisksCompleteFailed %s", err)) } func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -125,23 +119,20 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob if iVcpuCount != nil { vcpuCount, err = iVcpuCount.Int() if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("iVcpuCount.Int() fail %s", err)) return } } if iVmemSize != nil { vmemSize, err = iVmemSize.Int() if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("iVmemSize.Int fail %s", err)) return } } err = guest.GetDriver().RequestChangeVmConfig(ctx, guest, self, vcpuCount, vmemSize) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("guest.GetDriver().RequestChangeVmConfig fail %s", err)) return } var addCpu, addMem = 0, 0 @@ -167,15 +158,13 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob return nil }) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("Update fail %s", err)) return } var pendingUsage models.SQuota err = self.GetPendingUsage(&pendingUsage) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("GetPendingUsage %s", err)) return } // ownerCred := guest.GetOwnerUserCred() @@ -192,22 +181,19 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, guest.ProjectId, &pendingUsage, &cancelUsage) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("CancelPendingUsage fail %s", err)) return } err = self.SetPendingUsage(&pendingUsage) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("SetPendingUsage fail %s", err)) return } } self.SetStage("on_sync_status_complete", nil) err = guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) if err != nil { - self.markStageFailed(obj, ctx, err.Error()) - logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) + self.markStageFailed(ctx, guest, fmt.Sprintf("StartSyncstatus fail %s", err)) return } } @@ -221,8 +207,8 @@ func (self *GuestChangeConfigTask) OnSyncStatusComplete(ctx context.Context, obj } else { dt := jsonutils.NewDict() dt.Add(jsonutils.NewString(guest.Id), "id") - self.SetStageComplete(ctx, dt) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, "", self.UserCred, true) + self.SetStageComplete(ctx, dt) } } @@ -233,8 +219,9 @@ func (self *GuestChangeConfigTask) OnGuestStartComplete(ctx context.Context, obj self.SetStageComplete(ctx, dt) } -func (self *GuestChangeConfigTask) markStageFailed(obj db.IStandaloneModel, ctx context.Context, reason string) { - guest := obj.(*models.SGuest) +func (self *GuestChangeConfigTask) markStageFailed(ctx context.Context, guest *models.SGuest, reason string) { guest.SetStatus(self.UserCred, models.VM_CHANGE_FLAVOR_FAIL, reason) + db.OpsLog.LogEvent(guest, db.ACT_CHANGE_FLAVOR_FAIL, reason, self.UserCred) + logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, reason, self.UserCred, false) self.SetStageFailed(ctx, reason) } diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index 76502bc2e3..4afe217e10 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -105,7 +105,7 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d guest := obj.(*models.SGuest) if guest.Status == models.VM_RUNNING { self.SetStage("on_config_sync_complete", nil) - err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) + err := guest.StartSyncTask(ctx, self.UserCred, false, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) } @@ -184,15 +184,9 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex break } - ihost, err := guest.GetHost().GetIHost() - if err != nil { - self.SetStageFailed(ctx, "Host not found") - return - } - - iVM, e := ihost.GetIVMById(guest.GetExternalId()) + iVM, e := guest.GetIVM() if e != nil { - self.SetStageFailed(ctx, "Aliyun VM not found") + self.SetStageFailed(ctx, "iVM not found") return } @@ -205,7 +199,7 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex diskIndex += 1 } - if diskReady { + /*if diskReady { if guest.Status == models.VM_RUNNING { self.SetStage("on_config_sync_complete", nil) err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) @@ -213,16 +207,90 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex self.SetStageFailed(ctx, err.Error()) } } else { - self.SetStageComplete(ctx, nil) - } - } -} -func (self *ManagedGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + } + }*/ + self.SetStageComplete(ctx, nil) } -func (self *ManagedGuestCreateDiskTask) AttachManagedDisks(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { +/* +func (self *ManagedGuestCreateDiskTask) OnConfigSyncComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} +*/ + +type ESXiGuestCreateDiskTask struct { + SGuestBaseTask +} + +func (self *ESXiGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + host := guest.GetHost() + if host == nil { + self.SetStageFailed(ctx, "no valid host") + return + } + + diskIndex := 0 + for { + diskKey := fmt.Sprintf("disk.%d.id", diskIndex) + if !self.Params.Contains(diskKey) { + break + } + diskId, _ := self.Params.GetString(diskKey) + diskIndex += 1 + guestDisk := guest.GetGuestDisk(diskId) + if guestDisk == nil { + self.SetStageFailed(ctx, "fail to find guestdisk") + return + } + disk := guestDisk.GetDisk() + if disk == nil { + self.SetStageFailed(ctx, fmt.Sprintf("Disk %s not found", diskId)) + return + } + if disk.Status != models.DISK_INIT { + self.SetStageFailed(ctx, fmt.Sprintf("Disk %s already created??", diskId)) + return + } + ivm, err := guest.GetIVM() + if err != nil { + self.SetStageFailed(ctx, fmt.Sprintf("fail to find iVM for %s", guest.GetName())) + return + } + err = ivm.CreateDisk(ctx, disk.DiskSize, disk.Id, guestDisk.Driver) + if err != nil { + self.SetStageFailed(ctx, fmt.Sprintf("ivm.CreateDisk fail %s", guest.GetName())) + return + } + idisks, err := ivm.GetIDisks() + if err != nil { + self.SetStageFailed(ctx, fmt.Sprintf("ivm.GetIDisks fail %s", err)) + return + } + + log.Debugf("diskcount after create: %d", len(idisks)) + + vdisk := idisks[len(idisks)-1] + + _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { + disk.DiskSize = vdisk.GetDiskSizeMB() + disk.AccessPath = vdisk.GetAccessPath() + disk.ExternalId = vdisk.GetGlobalId() + return nil + }) + if err != nil { + self.SetStageFailed(ctx, fmt.Sprintf("disk.GetModelManager().TableSpec().Update fail %s", err)) + return + } + + disk.SetStatus(self.UserCred, models.DISK_READY, "create disk success") + disk.GetStorage().ClearSchedDescCache() + db.OpsLog.LogEvent(disk, db.ACT_ALLOCATE, disk.GetShortDesc(), self.UserCred) + db.OpsLog.LogAttachEvent(guest, disk, self.UserCred, disk.GetShortDesc()) + } + self.SetStageComplete(ctx, nil) } @@ -230,4 +298,5 @@ func init() { taskman.RegisterTask(GuestCreateDiskTask{}) taskman.RegisterTask(KVMGuestCreateDiskTask{}) taskman.RegisterTask(ManagedGuestCreateDiskTask{}) + taskman.RegisterTask(ESXiGuestCreateDiskTask{}) } diff --git a/pkg/compute/tasks/storage_cache_image_task.go b/pkg/compute/tasks/storage_cache_image_task.go index bd3d476fab..3b226aa9f7 100644 --- a/pkg/compute/tasks/storage_cache_image_task.go +++ b/pkg/compute/tasks/storage_cache_image_task.go @@ -46,8 +46,8 @@ func (self *StorageCacheImageTask) OnImageCacheComplete(ctx context.Context, obj storageCache := obj.(*models.SStoragecache) imageId, _ := self.Params.GetString("image_id") scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, storageCache.Id, imageId) - // extImgId, _ := data.GetString("image_id") - self.OnCacheSucc(ctx, storageCache, imageId, scimg) + extImgId, _ := data.GetString("image_id") + self.OnCacheSucc(ctx, storageCache, imageId, scimg, extImgId) } func (self *StorageCacheImageTask) OnImageCacheCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -55,14 +55,15 @@ func (self *StorageCacheImageTask) OnImageCacheCompleteFailed(ctx context.Contex imageId, _ := self.Params.GetString("image_id") scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, storageCache.Id, imageId) err := fmt.Errorf(data.String()) - self.OnCacheFailed(ctx, storageCache, imageId, scimg, err) + extImgId, _ := data.GetString("image_id") + self.OnCacheFailed(ctx, storageCache, imageId, scimg, err, extImgId) } -func (self *StorageCacheImageTask) OnCacheFailed(ctx context.Context, cache *models.SStoragecache, imageId string, scimg *models.SStoragecachedimage, err error) { +func (self *StorageCacheImageTask) OnCacheFailed(ctx context.Context, cache *models.SStoragecache, imageId string, scimg *models.SStoragecachedimage, err error, extImgId string) { scimg.SetStatus(self.UserCred, models.CACHED_IMAGE_STATUS_CACHE_FAILED, err.Error()) - /* if len(extImgId) > 0 && scimg.ExternalId != extImgId { + if len(extImgId) > 0 && scimg.ExternalId != extImgId { scimg.SetExternalId(extImgId) - }*/ + } body := jsonutils.NewDict() body.Add(jsonutils.NewString(err.Error()), "reason") body.Add(jsonutils.NewString(imageId), "image_id") @@ -70,8 +71,11 @@ func (self *StorageCacheImageTask) OnCacheFailed(ctx context.Context, cache *mod self.SetStageFailed(ctx, err.Error()) } -func (self *StorageCacheImageTask) OnCacheSucc(ctx context.Context, cache *models.SStoragecache, imageId string, scimg *models.SStoragecachedimage) { +func (self *StorageCacheImageTask) OnCacheSucc(ctx context.Context, cache *models.SStoragecache, imageId string, scimg *models.SStoragecachedimage, extImgId string) { scimg.SetStatus(self.UserCred, models.CACHED_IMAGE_STATUS_READY, "cached") + if len(extImgId) > 0 && scimg.ExternalId != extImgId { + scimg.SetExternalId(extImgId) + } models.CachedimageManager.ImageAddRefCount(imageId) db.OpsLog.LogEvent(cache, db.ACT_CACHED_IMAGE, imageId, self.UserCred) self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/storage_uncache_image_task.go b/pkg/compute/tasks/storage_uncache_image_task.go index 7e4e9743b9..63020d977a 100644 --- a/pkg/compute/tasks/storage_uncache_image_task.go +++ b/pkg/compute/tasks/storage_uncache_image_task.go @@ -2,11 +2,15 @@ package tasks import ( "context" + "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + + "yunion.io/x/onecloud/pkg/util/logclient" ) type StorageUncacheImageTask struct { @@ -23,15 +27,74 @@ func (self *StorageUncacheImageTask) OnInit(ctx context.Context, obj db.IStandal storageCache := obj.(*models.SStoragecache) + db.OpsLog.LogEvent(storageCache, db.ACT_UNCACHING_IMAGE, imageId, self.UserCred) + scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, storageCache.Id, imageId) - db.OpsLog.LogEvent(storageCache, db.ACT_UNCACHED_IMAGE, imageId, self.UserCred) + if scimg == nil || len(scimg.Path) == 0 { + // "image is not cached on this storage" + self.OnImageUncacheComplete(ctx, storageCache, nil) + } if isForce { - scimg.Detach(ctx, self.UserCred) - self.SetStageComplete(ctx, nil) + self.OnImageUncacheComplete(ctx, obj, data) return } - // TODO + host, err := storageCache.GetHost() + if err != nil { + self.OnTaskFailed(ctx, storageCache, fmt.Sprintf("fail to get host %s", err)) + return + } + + if host == nil { + self.OnImageUncacheComplete(ctx, obj, data) + return + } + + self.SetStage("OnImageUncacheComplete", nil) + + err = host.GetHostDriver().RequestUncacheImage(ctx, host, storageCache, self) + + if err != nil { + self.OnTaskFailed(ctx, storageCache, fmt.Sprintf("fail to uncache image %s", err)) + } +} + +func (self *StorageUncacheImageTask) OnTaskFailed(ctx context.Context, storageCache *models.SStoragecache, reason string) { + body := jsonutils.NewDict() + body.Add(jsonutils.NewString(reason), "reason") + imageId, _ := self.Params.GetString("image_id") + body.Add(jsonutils.NewString(imageId), "image_id") + + db.OpsLog.LogEvent(storageCache, db.ACT_UNCACHE_IMAGE_FAIL, body, self.UserCred) + + logclient.AddActionLog(storageCache, logclient.ACT_UNCACHED_IMAGE, body, self.UserCred, false) + + self.SetStageFailed(ctx, reason) +} + +func (self *StorageUncacheImageTask) OnImageUncacheCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + storageCache := obj.(*models.SStoragecache) + + self.OnTaskFailed(ctx, storageCache, data.String()) +} + +func (self *StorageUncacheImageTask) OnImageUncacheComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + log.Infof("Uncached image task success: %s", data) + storageCache := obj.(*models.SStoragecache) + + imageId, _ := self.Params.GetString("image_id") + scimg := models.StoragecachedimageManager.Register(ctx, self.UserCred, storageCache.Id, imageId) + if scimg != nil { + scimg.Detach(ctx, self.UserCred) + } + + body := jsonutils.NewDict() + body.Add(jsonutils.NewString(imageId), "image_id") + db.OpsLog.LogEvent(storageCache, db.ACT_UNCACHED_IMAGE, body, self.UserCred) + + logclient.AddActionLog(storageCache, db.ACT_UNCACHED_IMAGE, body, self.UserCred, true) + + self.SetStageComplete(ctx, nil) } diff --git a/pkg/mcclient/modules/managers.go b/pkg/mcclient/modules/managers.go index f1cc0ded38..62c7cb6d1a 100644 --- a/pkg/mcclient/modules/managers.go +++ b/pkg/mcclient/modules/managers.go @@ -155,4 +155,4 @@ func NewCloudmetaManager(keyword, keywordPlural string, columns, adminColumns [] adminColumns: adminColumns, serviceType: "cloudmeta"}, Keyword: keyword, KeywordPlural: keywordPlural} -} \ No newline at end of file +} diff --git a/pkg/mcclient/modules/resource.go b/pkg/mcclient/modules/resource.go index 35399fa2a2..65528750c4 100644 --- a/pkg/mcclient/modules/resource.go +++ b/pkg/mcclient/modules/resource.go @@ -297,7 +297,7 @@ func (this *ResourceManager) params2Body(s *mcclient.ClientSession, params jsonu return body } -func (this *ResourceManager)Create(session *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (this *ResourceManager) Create(session *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { return this.CreateInContexts(session, params, nil) } diff --git a/pkg/util/aliyun/aliyun.go b/pkg/util/aliyun/aliyun.go index 79fb5b905e..ddb092deb3 100644 --- a/pkg/util/aliyun/aliyun.go +++ b/pkg/util/aliyun/aliyun.go @@ -192,15 +192,3 @@ func (self *SAliyunClient) GetIStorageById(id string) (cloudprovider.ICloudStora } return nil, cloudprovider.ErrNotFound } - -func (self *SAliyunClient) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - for i := 0; i < len(self.iregions); i += 1 { - ihost, err := self.iregions[i].GetIStoragecacheById(id) - if err == nil { - return ihost, nil - } else if err != cloudprovider.ErrNotFound { - return nil, err - } - } - return nil, cloudprovider.ErrNotFound -} diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index c1189f130c..0f1507dd45 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -122,8 +122,8 @@ func (self *SDisk) Delete(ctx context.Context) error { return self.storage.zone.region.deleteDisk(self.DiskId) } -func (self *SDisk) Resize(ctx context.Context, size int64) error { - return self.storage.zone.region.resizeDisk(self.DiskId, size) +func (self *SDisk) Resize(ctx context.Context, sizeMb int64) error { + return self.storage.zone.region.resizeDisk(self.DiskId, sizeMb) } func (self *SDisk) GetName() string { @@ -262,14 +262,15 @@ func (self *SRegion) DeleteDisk(diskId string) error { return err } -func (self *SRegion) resizeDisk(diskId string, size int64) error { +func (self *SRegion) resizeDisk(diskId string, sizeMb int64) error { + sizeGb := sizeMb / 1024 params := make(map[string]string) params["DiskId"] = diskId - params["NewSize"] = fmt.Sprintf("%d", size) + params["NewSize"] = fmt.Sprintf("%d", sizeGb) _, err := self.ecsRequest("ResizeDisk", params) if err != nil { - log.Errorf("resizing disk (%s) to %d GiB failed: %s", diskId, size, err) + log.Errorf("resizing disk (%s) to %d GiB failed: %s", diskId, sizeGb, err) return err } @@ -377,3 +378,7 @@ func (self *SDisk) GetBillingType() string { func (self *SDisk) GetExpiredAt() time.Time { return self.ExpiredTime } + +func (self *SDisk) GetAccessPath() string { + return "" +} diff --git a/pkg/util/aliyun/image.go b/pkg/util/aliyun/image.go index 3f6947e89e..a34713576f 100644 --- a/pkg/util/aliyun/image.go +++ b/pkg/util/aliyun/image.go @@ -5,6 +5,7 @@ import ( "strings" "time" + "context" "github.com/aliyun/aliyun-oss-go-sdk/oss" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -85,7 +86,7 @@ func (self *SImage) IsEmulated() bool { return false } -func (self *SImage) Delete() error { +func (self *SImage) Delete(ctx context.Context) error { return self.storageCache.region.DeleteImage(self.ImageId) } diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 52aa481a3e..d352d2a869 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -467,7 +467,7 @@ func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd s return diskId, nil } -func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil) } @@ -861,3 +861,7 @@ func (self *SInstance) GetExpiredAt() time.Time { func (self *SInstance) UpdateUserData(userData string) error { return self.host.zone.region.updateInstance(self.InstanceId, "", "", "", "", userData) } + +func (self *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { + return cloudprovider.ErrNotSupported +} diff --git a/pkg/util/aliyun/provider/provider.go b/pkg/util/aliyun/provider/provider.go index fe5a4a1813..6b3d2efa99 100644 --- a/pkg/util/aliyun/provider/provider.go +++ b/pkg/util/aliyun/provider/provider.go @@ -83,24 +83,8 @@ func (self *SAliyunProvider) GetIRegions() []cloudprovider.ICloudRegion { return self.client.GetIRegions() } -func (self *SAliyunProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) { - return self.client.GetIRegionById(id) -} - -func (self *SAliyunProvider) GetIHostById(id string) (cloudprovider.ICloudHost, error) { - return self.client.GetIHostById(id) -} - -func (self *SAliyunProvider) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { - return self.client.GetIVpcById(id) -} - -func (self *SAliyunProvider) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return self.client.GetIStorageById(id) -} - -func (self *SAliyunProvider) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - return self.client.GetIStoragecacheById(id) +func (self *SAliyunProvider) GetIRegionById(extId string) (cloudprovider.ICloudRegion, error) { + return self.client.GetIRegionById(extId) } func (self *SAliyunProvider) GetBalance() (float64, error) { @@ -111,6 +95,6 @@ func (self *SAliyunProvider) GetBalance() (float64, error) { return balance.AvailableAmount, nil } -func (self *SAliyunProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { +func (self *SAliyunProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index fecb4d90cb..42e0cecec3 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -550,6 +550,40 @@ func (self *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, er return nil, cloudprovider.ErrNotFound } +func (self *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) { + iHosts := make([]cloudprovider.ICloudHost, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneHost, err := izones[i].GetIHosts() + if err != nil { + return nil, err + } + iHosts = append(iHosts, iZoneHost...) + } + return iHosts, nil +} + +func (self *SRegion) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + iStores := make([]cloudprovider.ICloudStorage, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneStores, err := izones[i].GetIStorages() + if err != nil { + return nil, err + } + iStores = append(iStores, iZoneStores...) + } + return iStores, nil +} + func (self *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { storageCache := self.getStoragecache() if storageCache.GetGlobalId() == id { diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index d8dec31739..26c1a729c2 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -118,7 +118,7 @@ func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudpr return disk, nil } -func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { +func (self *SStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { if disk, err := self.zone.region.getDisk(idStr); err != nil { return nil, err } else { @@ -126,3 +126,7 @@ func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { return disk, nil } } + +func (self *SStorage) GetMountPoint() string { + return "" +} diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go index 231d5d3623..a1318f5bd0 100644 --- a/pkg/util/aliyun/storagecache.go +++ b/pkg/util/aliyun/storagecache.go @@ -87,6 +87,22 @@ func (self *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { return self.iimages, nil } +func (self *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { + parts, _, err := self.region.GetImages(ImageStatusType(""), ImageOwnerSelf, []string{extId}, "", 0, 1) + if err != nil { + return nil, err + } + if len(parts) == 0 { + return nil, cloudprovider.ErrNotFound + } + parts[0].storageCache = self + return &parts[0], nil +} + +func (self *SStoragecache) GetPath() string { + return "" +} + func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, extId string, isForce bool) (string, error) { if len(extId) > 0 { diff --git a/pkg/util/aws/aws.go b/pkg/util/aws/aws.go index cbdfc5bf06..65224a5c37 100644 --- a/pkg/util/aws/aws.go +++ b/pkg/util/aws/aws.go @@ -185,18 +185,6 @@ func (self *SAwsClient) GetIStorageById(id string) (cloudprovider.ICloudStorage, return nil, cloudprovider.ErrNotFound } -func (self *SAwsClient) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - for i := 0; i < len(self.iregions); i += 1 { - ihost, err := self.iregions[i].GetIStoragecacheById(id) - if err == nil { - return ihost, nil - } else if err != cloudprovider.ErrNotFound { - return nil, err - } - } - return nil, cloudprovider.ErrNotFound -} - type SAccountBalance struct { AvailableAmount float64 AvailableCashAmount float64 diff --git a/pkg/util/aws/disk.go b/pkg/util/aws/disk.go index f11e2912bc..20308db644 100644 --- a/pkg/util/aws/disk.go +++ b/pkg/util/aws/disk.go @@ -215,8 +215,8 @@ func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return isnapshots, nil } -func (self *SDisk) Resize(ctx context.Context, newSize int64) error { - return self.storage.zone.region.resizeDisk(self.DiskId, newSize) +func (self *SDisk) Resize(ctx context.Context, newSizeMb int64) error { + return self.storage.zone.region.resizeDisk(self.DiskId, newSizeMb) } func (self *SDisk) Reset(ctx context.Context, snapshotId string) error { @@ -344,14 +344,15 @@ func (self *SRegion) DeleteDisk(diskId string) error { return err } -func (self *SRegion) resizeDisk(diskId string, size int64) error { +func (self *SRegion) resizeDisk(diskId string, sizeMb int64) error { // https://docs.aws.amazon.com/zh_cn/AWSEC2/latest/UserGuide/volume_constraints.html // MBR -> 2 TiB // GPT -> 16 TiB // size unit GiB + sizeGb := sizeMb / 1024 params := &ec2.ModifyVolumeInput{} - if size > 0 { - params.SetSize(size) + if sizeGb > 0 { + params.SetSize(sizeGb) } else { return fmt.Errorf("size should great than 0") } @@ -440,3 +441,7 @@ func (self *SRegion) CreateDisk(zoneId string, category string, name string, siz } return StrVal(ret.VolumeId), nil } + +func (disk *SDisk) GetAccessPath() string { + return "" +} diff --git a/pkg/util/aws/image.go b/pkg/util/aws/image.go index 700e43692b..d77deabae1 100644 --- a/pkg/util/aws/image.go +++ b/pkg/util/aws/image.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "context" "github.com/aws/aws-sdk-go/service/ec2" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -115,7 +116,7 @@ func (self *SImage) GetMetadata() *jsonutils.JSONDict { return data } -func (self *SImage) Delete() error { +func (self *SImage) Delete(ctx context.Context) error { // todo: implement me return self.storageCache.region.DeleteImage(self.ImageId) } diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go index de1e3c9f48..4d72a804de 100644 --- a/pkg/util/aws/instance.go +++ b/pkg/util/aws/instance.go @@ -369,7 +369,7 @@ func (self *SInstance) DeployVM(ctx context.Context, name string, password strin return self.host.zone.region.DeployVM(self.InstanceId, name, password, publicKey, deleteKeypair, description) } -func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { return self.host.zone.region.ChangeVMConfig(self.ZoneId, self.InstanceId, ncpu, vmem, nil) } @@ -819,3 +819,7 @@ func (self *SRegion) AttachDisk(instanceId string, diskId string, deviceName str _, err := self.ec2Client.AttachVolume(params) return err } + +func (self *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { + return cloudprovider.ErrNotSupported +} diff --git a/pkg/util/aws/provider/provider.go b/pkg/util/aws/provider/provider.go index 20c6327245..fc44072ac3 100644 --- a/pkg/util/aws/provider/provider.go +++ b/pkg/util/aws/provider/provider.go @@ -66,22 +66,6 @@ func (self *SAwsProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, return self.client.GetIRegionById(id) } -func (self *SAwsProvider) GetIHostById(id string) (cloudprovider.ICloudHost, error) { - return self.client.GetIHostById(id) -} - -func (self *SAwsProvider) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { - return self.client.GetIVpcById(id) -} - -func (self *SAwsProvider) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return self.client.GetIStorageById(id) -} - -func (self *SAwsProvider) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - return self.client.GetIStoragecacheById(id) -} - func (self *SAwsProvider) GetBalance() (float64, error) { balance, err := self.client.QueryAccountBalance() if err != nil { @@ -90,6 +74,6 @@ func (self *SAwsProvider) GetBalance() (float64, error) { return balance.AvailableAmount, nil } -func (self *SAwsProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { +func (self *SAwsProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go index 910bae7767..3206112667 100644 --- a/pkg/util/aws/region.go +++ b/pkg/util/aws/region.go @@ -348,6 +348,40 @@ func (self *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, er return nil, cloudprovider.ErrNotFound } +func (self *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) { + iHosts := make([]cloudprovider.ICloudHost, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneHost, err := izones[i].GetIHosts() + if err != nil { + return nil, err + } + iHosts = append(iHosts, iZoneHost...) + } + return iHosts, nil +} + +func (self *SRegion) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + iStores := make([]cloudprovider.ICloudStorage, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneStores, err := izones[i].GetIStorages() + if err != nil { + return nil, err + } + iStores = append(iStores, iZoneStores...) + } + return iStores, nil +} + func (self *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { if self.storageCache == nil { self.storageCache = &SStoragecache{region: self} diff --git a/pkg/util/aws/storage.go b/pkg/util/aws/storage.go index 35d7081df8..937598edcc 100644 --- a/pkg/util/aws/storage.go +++ b/pkg/util/aws/storage.go @@ -114,7 +114,7 @@ func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudpr return disk, nil } -func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { +func (self *SStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { if disk, err := self.zone.region.GetDisk(idStr); err != nil { return nil, err } else { @@ -122,3 +122,7 @@ func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { return disk, nil } } + +func (self *SStorage) GetMountPoint() string { + return "" +} diff --git a/pkg/util/aws/storagecache.go b/pkg/util/aws/storagecache.go index b1a42471bd..c43e88ec6f 100644 --- a/pkg/util/aws/storagecache.go +++ b/pkg/util/aws/storagecache.go @@ -61,6 +61,22 @@ func (self *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { return self.iimages, nil } +func (self *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { + parts, _, err := self.region.GetImages(ImageStatusType(""), ImageOwnerSelf, []string{extId}, "", 0, 1) + if err != nil { + return nil, err + } + if len(parts) == 0 { + return nil, cloudprovider.ErrNotFound + } + parts[0].storageCache = self + return &parts[0], nil +} + +func (self *SStoragecache) GetPath() string { + return "" +} + func (self *SStoragecache) GetManagerId() string { return self.region.client.providerId } diff --git a/pkg/util/azure/azure.go b/pkg/util/azure/azure.go index ad7268feb6..95e08cd7ea 100644 --- a/pkg/util/azure/azure.go +++ b/pkg/util/azure/azure.go @@ -707,18 +707,6 @@ func (self *SAzureClient) GetIStorageById(id string) (cloudprovider.ICloudStorag return nil, cloudprovider.ErrNotFound } -func (self *SAzureClient) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - for i := 0; i < len(self.iregions); i += 1 { - ihost, err := self.iregions[i].GetIStoragecacheById(id) - if err == nil { - return ihost, nil - } else if err != cloudprovider.ErrNotFound { - return nil, err - } - } - return nil, cloudprovider.ErrNotFound -} - type SAccountBalance struct { AvailableAmount float64 AvailableCashAmount float64 diff --git a/pkg/util/azure/classic_disk.go b/pkg/util/azure/classic_disk.go index 3ead009503..895a937bc3 100644 --- a/pkg/util/azure/classic_disk.go +++ b/pkg/util/azure/classic_disk.go @@ -209,6 +209,10 @@ func (self *SClassicDisk) Reset(ctx context.Context, snapshotId string) error { return cloudprovider.ErrNotSupported } -func (self *SClassicDisk) Resize(ctx context.Context, size int64) error { +func (self *SClassicDisk) Resize(ctx context.Context, sizeMb int64) error { return cloudprovider.ErrNotSupported } + +func (disk *SClassicDisk) GetAccessPath() string { + return "" +} diff --git a/pkg/util/azure/classic_instance.go b/pkg/util/azure/classic_instance.go index a53a3da659..6a7006c578 100644 --- a/pkg/util/azure/classic_instance.go +++ b/pkg/util/azure/classic_instance.go @@ -270,11 +270,8 @@ func (self *SClassicInstance) DetachDisk(ctx context.Context, diskId string) err return cloudprovider.WaitStatus(self, self.GetStatus(), 10*time.Second, 300*time.Second) } -func (self *SClassicInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { - if err := self.host.zone.region.ChangeVMConfig(ctx, instanceId, ncpu, vmem); err != nil { - return err - } - return cloudprovider.WaitStatus(self, self.GetStatus(), 10*time.Second, 300*time.Second) +func (self *SClassicInstance) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { + return cloudprovider.ErrNotImplemented } func (self *SClassicInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { @@ -462,3 +459,7 @@ func (self *SClassicInstance) GetExpiredAt() time.Time { func (self *SClassicInstance) UpdateUserData(userData string) error { return cloudprovider.ErrNotSupported } + +func (self *SClassicInstance) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { + return cloudprovider.ErrNotSupported +} diff --git a/pkg/util/azure/classic_storage.go b/pkg/util/azure/classic_storage.go index 7f33ce8012..329f965637 100644 --- a/pkg/util/azure/classic_storage.go +++ b/pkg/util/azure/classic_storage.go @@ -66,7 +66,7 @@ func (self *SClassicStorage) CreateIDisk(name string, sizeGb int, desc string) ( return nil, cloudprovider.ErrNotImplemented } -func (self *SClassicStorage) GetIDisk(diskId string) (cloudprovider.ICloudDisk, error) { +func (self *SClassicStorage) GetIDiskById(diskId string) (cloudprovider.ICloudDisk, error) { disks, err := self.GetIDisks() if err != nil { return nil, err @@ -125,3 +125,7 @@ func (self *SClassicStorage) Refresh() error { // do nothing return nil } + +func (self *SClassicStorage) GetMountPoint() string { + return "" +} diff --git a/pkg/util/azure/disk.go b/pkg/util/azure/disk.go index 06cb70e2fb..57b7490183 100644 --- a/pkg/util/azure/disk.go +++ b/pkg/util/azure/disk.go @@ -173,8 +173,8 @@ func (self *SDisk) Delete(ctx context.Context) error { return self.storage.zone.region.deleteDisk(self.ID) } -func (self *SDisk) Resize(ctx context.Context, size int64) error { - return self.storage.zone.region.ResizeDisk(self.ID, int32(size)) +func (self *SDisk) Resize(ctx context.Context, sizeMb int64) error { + return self.storage.zone.region.ResizeDisk(self.ID, int32(sizeMb/1024)) } func (self *SDisk) GetName() string { @@ -321,3 +321,7 @@ func (self *SDisk) Reset(ctx context.Context, snapshotId string) error { func (self *SRegion) resetDisk(diskId, snapshotId string) error { return cloudprovider.ErrNotSupported } + +func (disk *SDisk) GetAccessPath() string { + return "" +} diff --git a/pkg/util/azure/image.go b/pkg/util/azure/image.go index a5fe3a2c30..8c53edcefd 100644 --- a/pkg/util/azure/image.go +++ b/pkg/util/azure/image.go @@ -3,6 +3,7 @@ package azure import ( "strings" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -141,6 +142,20 @@ func (self *SRegion) GetImageByName(name string) (*SImage, error) { return nil, cloudprovider.ErrNotFound } +func (self *SRegion) GetImageById(idstr string) (*SImage, error) { + images := []SImage{} + err := self.client.ListAll("Microsoft.Compute/images", &images) + if err != nil { + return nil, err + } + for i := 0; i < len(images); i++ { + if images[i].ID == idstr { + return &images[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + func (self *SRegion) CreateImageByBlob(imageName, osType, blobURI string, diskSizeGB int32) (*SImage, error) { if diskSizeGB < 1 || diskSizeGB > 4095 { diskSizeGB = 30 @@ -207,7 +222,7 @@ func (self *SImage) GetBlobUri() string { return self.Properties.StorageProfile.OsDisk.BlobURI } -func (self *SImage) Delete() error { +func (self *SImage) Delete(ctx context.Context) error { return self.storageCache.region.DeleteImage(self.ID) } diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index b9aaf9d93f..7effd68a8f 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -548,7 +548,7 @@ func (region *SRegion) DetachDisk(instanceId, diskId string) error { return region.client.Update(jsonutils.Marshal(instance), nil) } -func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { for _, vmSize := range self.host.zone.region.getHardwareProfile(ncpu, vmem) { self.Properties.HardwareProfile.VMSize = vmSize self.Properties.ProvisioningState = "" @@ -567,7 +567,7 @@ func (region *SRegion) ChangeVMConfig(ctx context.Context, instanceId string, nc if err != nil { return err } - return instacen.ChangeConfig(ctx, instanceId, ncpu, vmem) + return instacen.ChangeConfig(ctx, ncpu, vmem) } func (self *SInstance) DeployVM(ctx context.Context, name string, password string, publicKey string, deleteKeypair bool, description string) error { @@ -1041,3 +1041,7 @@ func (self *SInstance) GetExpiredAt() time.Time { func (self *SInstance) UpdateUserData(userData string) error { return cloudprovider.ErrNotSupported } + +func (self *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { + return cloudprovider.ErrNotSupported +} diff --git a/pkg/util/azure/provider/provider.go b/pkg/util/azure/provider/provider.go index 214877edd3..a4925b2395 100644 --- a/pkg/util/azure/provider/provider.go +++ b/pkg/util/azure/provider/provider.go @@ -67,22 +67,6 @@ func (self *SAzureProvider) GetIRegionById(id string) (cloudprovider.ICloudRegio return self.client.GetIRegionById(id) } -func (self *SAzureProvider) GetIHostById(id string) (cloudprovider.ICloudHost, error) { - return self.client.GetIHostById(id) -} - -func (self *SAzureProvider) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { - return self.client.GetIVpcById(id) -} - -func (self *SAzureProvider) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return self.client.GetIStorageById(id) -} - -func (self *SAzureProvider) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - return self.client.GetIStoragecacheById(id) -} - func (self *SAzureProvider) GetBalance() (float64, error) { balance, err := self.client.QueryAccountBalance() if err != nil { @@ -91,6 +75,6 @@ func (self *SAzureProvider) GetBalance() (float64, error) { return balance.AvailableAmount, nil } -func (self *SAzureProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { +func (self *SAzureProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/util/azure/region.go b/pkg/util/azure/region.go index ff634bdb8c..fbba907727 100644 --- a/pkg/util/azure/region.go +++ b/pkg/util/azure/region.go @@ -175,6 +175,40 @@ func (self *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, er return nil, cloudprovider.ErrNotFound } +func (self *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) { + iHosts := make([]cloudprovider.ICloudHost, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneHost, err := izones[i].GetIHosts() + if err != nil { + return nil, err + } + iHosts = append(iHosts, iZoneHost...) + } + return iHosts, nil +} + +func (self *SRegion) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + iStores := make([]cloudprovider.ICloudStorage, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneStores, err := izones[i].GetIStorages() + if err != nil { + return nil, err + } + iStores = append(iStores, iZoneStores...) + } + return iStores, nil +} + func (self *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { storageCache := self.getStoragecache() if storageCache.GetGlobalId() == id { diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go index 660b3bd69d..964a501fb8 100644 --- a/pkg/util/azure/storage.go +++ b/pkg/util/azure/storage.go @@ -68,7 +68,7 @@ func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudpr return disk, nil } -func (self *SStorage) GetIDisk(diskId string) (cloudprovider.ICloudDisk, error) { +func (self *SStorage) GetIDiskById(diskId string) (cloudprovider.ICloudDisk, error) { if disk, err := self.zone.region.GetDisk(diskId); err != nil { return nil, err } else { @@ -157,3 +157,7 @@ func (self *SStorage) Refresh() error { // do nothing return nil } + +func (self *SStorage) GetMountPoint() string { + return "" +} diff --git a/pkg/util/azure/storagecache.go b/pkg/util/azure/storagecache.go index 0715ea4393..d5f1955237 100644 --- a/pkg/util/azure/storagecache.go +++ b/pkg/util/azure/storagecache.go @@ -84,6 +84,19 @@ func (self *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { return self.iimages, nil } +func (self *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { + img, err := self.region.GetImageById(extId) + if err != nil { + return nil, err + } + img.storageCache = self + return img, nil +} + +func (self *SStoragecache) GetPath() string { + return "" +} + func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, extId string, isForce bool) (string, error) { if len(extId) > 0 { log.Debugf("UploadImage: Image external ID exists %s", extId) diff --git a/pkg/util/esxi/datacenter.go b/pkg/util/esxi/datacenter.go index 652facd020..1a6a5c0124 100644 --- a/pkg/util/esxi/datacenter.go +++ b/pkg/util/esxi/datacenter.go @@ -1,7 +1,10 @@ package esxi import ( + "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -26,6 +29,10 @@ func (dc *SDatacenter) getDatacenter() *mo.Datacenter { return dc.object.(*mo.Datacenter) } +func (dc *SDatacenter) getObjectDatacenter() *object.Datacenter { + return object.NewDatacenter(dc.manager.client.Client, dc.object.Reference()) +} + func (dc *SDatacenter) scanHosts() error { if dc.ihosts == nil { var hosts []mo.HostSystem @@ -51,15 +58,11 @@ func (dc *SDatacenter) GetIHosts() ([]cloudprovider.ICloudHost, error) { func (dc *SDatacenter) scanDatastores() error { if dc.istorages == nil { - stores := make([]mo.Datastore, 0) + var stores []mo.Datastore dsList := dc.getDatacenter().Datastore - for i := 0; i < len(dsList); i += 1 { - var ds mo.Datastore - err := dc.manager.reference2Object(dsList[i], DATASTORE_PROPS, &ds) - if err != nil { - return err - } - stores = append(stores, ds) + err := dc.manager.references2Objects(dsList, DATASTORE_PROPS, &stores) + if err != nil { + return err } dc.istorages = make([]cloudprovider.ICloudStorage, len(stores)) for i := 0; i < len(stores); i += 1 { @@ -102,3 +105,22 @@ func (dc *SDatacenter) GetIStorageByMoId(idstr string) (cloudprovider.ICloudStor } return nil, cloudprovider.ErrNotFound } + +func (dc *SDatacenter) getDcObj() *object.Datacenter { + return object.NewDatacenter(dc.manager.client.Client, dc.object.Reference()) +} + +func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference) ([]cloudprovider.ICloudVM, error) { + var vms []mo.VirtualMachine + err := dc.manager.references2Objects(vmRefs, VIRTUAL_MACHINE_PROPS, &vms) + if err != nil { + log.Errorf("references2Objects fail %s", err) + return nil, err + } + + retVms := make([]cloudprovider.ICloudVM, len(vms)) + for i := 0; i < len(vms); i += 1 { + retVms[i] = NewVirtualMachine(dc.manager, &vms[i], dc) + } + return retVms, nil +} diff --git a/pkg/util/esxi/devtools.go b/pkg/util/esxi/devtools.go new file mode 100644 index 0000000000..ec66271fd7 --- /dev/null +++ b/pkg/util/esxi/devtools.go @@ -0,0 +1,43 @@ +package esxi + +import ( + "github.com/vmware/govmomi/vim25/types" +) + +func NewDiskDev(sizeMb int64, templatePath string, uuid string, index int32, key int32, controlKey int32) *types.VirtualDisk { + device := types.VirtualDisk{} + + var backFile *types.VirtualDiskFlatVer2BackingInfo + if len(templatePath) > 0 { + backFile = &types.VirtualDiskFlatVer2BackingInfo{} + backFile.FileName = templatePath + } + + diskFile := types.VirtualDiskFlatVer2BackingInfo{} + diskFile.DiskMode = "persistent" + thinProvisioned := true + diskFile.ThinProvisioned = &thinProvisioned + diskFile.Uuid = uuid + if backFile != nil { + diskFile.Parent = backFile + } + + device.Backing = &diskFile + + if sizeMb > 0 { + device.CapacityInKB = sizeMb * 1024 + } + + device.ControllerKey = controlKey + device.Key = key + index + device.UnitNumber = &index + + return &device +} + +func addDevSpec(device types.BaseVirtualDevice) *types.VirtualDeviceConfigSpec { + spec := types.VirtualDeviceConfigSpec{} + spec.Operation = types.VirtualDeviceConfigSpecOperationAdd + spec.Device = device + return &spec +} diff --git a/pkg/util/esxi/fakeregion.go b/pkg/util/esxi/fakeregion.go new file mode 100644 index 0000000000..da0fe948cd --- /dev/null +++ b/pkg/util/esxi/fakeregion.go @@ -0,0 +1,69 @@ +package esxi + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func (cli *SESXiClient) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (cli *SESXiClient) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (cli *SESXiClient) GetIHosts() ([]cloudprovider.ICloudHost, error) { + dcs, err := cli.GetDatacenters() + if err != nil { + return nil, err + } + + ihosts := make([]cloudprovider.ICloudHost, 0) + for i := 0; i < len(dcs); i += 1 { + dcIHosts, err := dcs[i].GetIHosts() + if err != nil { + return nil, err + } + ihosts = append(ihosts, dcIHosts...) + } + return ihosts, nil +} + +func (cli *SESXiClient) GetIHostById(id string) (cloudprovider.ICloudHost, error) { + return cli.FindHostByIp(id) +} + +func (cli *SESXiClient) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + dcs, err := cli.GetDatacenters() + if err != nil { + return nil, err + } + + iStorages := make([]cloudprovider.ICloudStorage, 0) + for i := 0; i < len(dcs); i += 1 { + dcIStorages, err := dcs[i].GetIStorages() + if err != nil { + return nil, err + } + iStorages = append(iStorages, dcIStorages...) + } + return iStorages, nil +} + +func (cli *SESXiClient) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + iStorages, err := cli.GetIStorages() + if err != nil { + return nil, err + } + for i := 0; i < len(iStorages); i += 1 { + if iStorages[i].GetGlobalId() == id { + return iStorages[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (cli *SESXiClient) GetProvider() string { + return models.CLOUD_PROVIDER_VMWARE +} diff --git a/pkg/util/esxi/guestos.go b/pkg/util/esxi/guestos.go new file mode 100644 index 0000000000..a5c77f185b --- /dev/null +++ b/pkg/util/esxi/guestos.go @@ -0,0 +1,211 @@ +package esxi + +type TOSType string + +type TOSArch string + +const ( + LINUX = TOSType("Linux") + WINDOWS = TOSType("Windows") + MACOS = TOSType("macOS") + FREEBSD = TOSType("FreeBSD") + SOLARIS = TOSType("Solaris") + VMWARE = TOSType("VMware") + + X86 = TOSArch("x86") + X86_64 = TOSArch("x86_64") +) + +type SOsInfo struct { + OsType TOSType + OsDistribution string + OsVersion string + OsArch TOSArch +} + +func asian(ver string, arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Asianux Server", ver, arch} +} + +func centos(arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "CentOS", "4/5", arch} +} + +func macos(ver string, arch TOSArch) SOsInfo { + return SOsInfo{MACOS, "Mac OS", ver, arch} +} + +func debian(ver string, arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Debian", ver, arch} + +} + +func fedora(ver string, arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Fedora", ver, arch} +} + +func freebsd(arch TOSArch) SOsInfo { + return SOsInfo{FREEBSD, "FreeBSD", "", arch} +} + +func rhel(ver string, arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "RedHat Enterprise Linux", ver, arch} + +} + +func suse(ver string, arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "SuSE", ver, arch} +} + +func opensuse(arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "OpenSuSE", "?", arch} + +} + +func oracle(ver string, arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Oracle", ver, arch} +} + +func linux(ver string, arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Generic", ver, arch} +} + +func windows(dist string, arch TOSArch) SOsInfo { + return SOsInfo{WINDOWS, dist, "", arch} +} + +func solaris(ver string, arch TOSArch) SOsInfo { + return SOsInfo{SOLARIS, "Solaris", ver, arch} +} + +func turbo(arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Turbo Linux", "?", arch} +} + +func ubuntu(arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Ubuntu", "?", arch} +} + +func mandriva(arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Mandriva", "?", arch} +} + +func mandrake(arch TOSArch) SOsInfo { + return SOsInfo{LINUX, "Mandrake", "?", arch} +} + +func vmware(ver string) SOsInfo { + return SOsInfo{VMWARE, "ESX", ver, X86_64} +} + +var ( + GuestOsInfo = map[string]SOsInfo{ + "asianux3_64Guest": asian("3", X86_64), + "asianux3Guest": asian("3", X86), + "asianux4_64Guest": asian("4", X86_64), + "asianux4Guest": asian("4", X86), + "centos64Guest": centos(X86_64), + "centosGuest": centos(X86), + "darwin10_64Guest": macos("10.6", X86_64), + "darwin10Guest": macos("10.6", X86), + "darwin11_64Guest": macos("10.7", X86_64), + "darwin11Guest": macos("10.7", X86), + "darwin12_64Guest": macos("10.8", X86_64), + "darwin13_64Guest": macos("10.9", X86_64), + "darwin64Guest": macos("10.5", X86_64), + "darwinGuest": macos("10.5", X86), + "debian4_64Guest": debian("4", X86_64), + "debian4Guest": debian("4", X86), + "debian5_64Guest": debian("5", X86_64), + "debian5Guest": debian("5", X86), + "debian6_64Guest": debian("6", X86_64), + "debian6Guest": debian("6", X86), + "debian7_64Guest": debian("7", X86_64), + "debian7Guest": debian("7", X86), + "fedora64Guest": fedora("?", X86_64), + "fedoraGuest": fedora("?", X86), + "freebsd64Guest": freebsd(X86_64), + "freebsdGuest": freebsd(X86), + "genericLinuxGuest": linux("?", X86), + "mandrakeGuest": mandrake(X86), + "mandriva64Guest": mandriva(X86_64), + "mandrivaGuest": mandriva(X86), + "opensuse64Guest": opensuse(X86_64), + "opensuseGuest": opensuse(X86), + "oracleLinux64Guest": oracle("4/5", X86_64), + "oracleLinuxGuest": oracle("4/5", X86), + "other24xLinux64Guest": linux("2.4", X86_64), + "other24xLinuxGuest": linux("2.4", X86), + "other26xLinux64Guest": linux("2.6", X86_64), + "other26xLinuxGuest": linux("2.6", X86), + "other3xLinux64Guest": linux("3.x", X86_64), + "other3xLinuxGuest": linux("3.x", X86_64), + "otherLinux64Guest": linux("2.2", X86_64), + "otherLinuxGuest": linux("2.2", X86), + "redhatGuest": rhel("2.1", X86), + "rhel2Guest": rhel("2", X86), + "rhel3_64Guest": rhel("3", X86_64), + "rhel3Guest": rhel("3", X86), + "rhel4_64Guest": rhel("4", X86_64), + "rhel4Guest": rhel("4", X86), + "rhel5_64Guest": rhel("5", X86_64), + "rhel5Guest": rhel("5", X86), + "rhel6_64Guest": rhel("6", X86_64), + "rhel6Guest": rhel("6", X86), + "rhel7_64Guest": rhel("7", X86_64), + "rhel7Guest": rhel("7", X86), + "sles10_64Guest": suse("10", X86_64), + "sles10Guest": suse("10", X86), + "sles11_64Guest": suse("11", X86_64), + "sles11Guest": suse("11", X86), + "sles12_64Guest": suse("12", X86_64), + "sles12Guest": suse("12", X86), + "sles64Guest": suse("9", X86_64), + "slesGuest": suse("9", X86), + "solaris10_64Guest": solaris("10", X86_64), + "solaris10Guest": solaris("10", X86), + "solaris11_64Guest": solaris("11", X86_64), + "solaris6Guest": solaris("6", X86), + "solaris7Guest": solaris("7", X86), + "solaris8Guest": solaris("8", X86), + "solaris9Guest": solaris("9", X86), + "suse64Guest": suse("?", X86_64), + "suseGuest": suse("?", X86), + "turboLinux64Guest": turbo(X86_64), + "turboLinuxGuest": turbo(X86), + "ubuntu64Guest": ubuntu(X86_64), + "ubuntuGuest": ubuntu(X86), + "vmkernel5Guest": vmware("5"), + "vmkernelGuest": vmware("4"), + "win2000AdvServGuest": windows("Windows 2000 Advanced Server", X86), + "win2000ProGuest": windows("Windows 2000 Professional", X86), + "win2000ServGuest": windows("Windows 2000 Server", X86), + "win31Guest": windows("Windows 3.1", X86), + "win95Guest": windows("Windows 95", X86), + "win98Guest": windows("Windows 98", X86), + "windows7_64Guest": windows("Windows 7", X86_64), + "windows7Guest": windows("Windows 7", X86_64), + "windows7Server64Guest": windows("Windows Server 2008 R2", X86_64), + "windows8_64Guest": windows("Windows 8", X86_64), + "windows8Guest": windows("Windows 8", X86), + "windows8Server64Guest": windows("Windows 8 Server", X86_64), + "windowsHyperVGuest": windows("Windows Hyper-V", X86_64), + "winLonghorn64Guest": windows("Windows Longhorn", X86_64), + "winLonghornGuest": windows("Windows Longhorn", X86), + "winMeGuest": windows("Windows Millenium Edition", X86), + "winNetBusinessGuest": windows("Windows Small Business Server 2003", X86), + "winNetDatacenter64Guest": windows("Windows Server 2003 Datacenter Edition", X86_64), + "winNetDatacenterGuest": windows("Windows Server 2003 Datacenter Edition", X86), + "winNetEnterprise64Guest": windows("Windows Server 2003 Enterprise Edition", X86_64), + "winNetEnterpriseGuest": windows("Windows Server 2003 Enterprise Edition", X86), + "winNetStandard64Guest": windows("Windows Server 2003 Standard Edition", X86_64), + "winNetStandardGuest": windows("Windows Server 2003 Standard Edition", X86), + "winNetWebGuest": windows("Windows Server 2003 Web Edition", X86), + "winNTGuest": windows("Windows NT 4", X86), + "winVista64Guest": windows("Windows Vista", X86_64), + "winVistaGuest": windows("Windows Vista", X86), + "winXPHomeGuest": windows("Windows XP Home Edition", X86), + "winXPPro64Guest": windows("Windows XP Professional Edition", X86_64), + "winXPProGuest": windows("Windows XP Professional", X86), + } +) diff --git a/pkg/util/esxi/host.go b/pkg/util/esxi/host.go index fc4581fb9e..b387b8306e 100644 --- a/pkg/util/esxi/host.go +++ b/pkg/util/esxi/host.go @@ -1,17 +1,19 @@ package esxi import ( + "context" + "fmt" + "strings" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" "yunion.io/x/jsonutils" - - "fmt" - "github.com/vmware/govmomi/vim25/types" - "strings" "yunion.io/x/log" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/util/netutils" ) var HOST_SYSTEM_PROPS = []string{"name", "parent", "summary", "config", "hardware", "vm", "datastore"} @@ -65,6 +67,8 @@ type SHost struct { datastores []cloudprovider.ICloudStorage + storageCache *SDatastoreImageCache + vms []cloudprovider.ICloudVM } @@ -102,7 +106,15 @@ func (self *SHost) GetStatus() string { } func (self *SHost) Refresh() error { - self.vms = nil + base := self.SManagedObject + var moObj mo.HostSystem + err := self.manager.reference2Object(self.object.Reference(), HOST_SYSTEM_PROPS, &moObj) + if err != nil { + return err + } + base.object = &moObj + *self = SHost{} + self.SManagedObject = base return nil } @@ -114,21 +126,23 @@ func (self *SHost) fetchVMs() error { if self.vms != nil { return nil } - var vms []mo.VirtualMachine - err := self.manager.references2Objects(self.getHostSystem().Vm, VIRTUAL_MACHINE_PROPS, &vms) - if err != nil { - return err - } dc, err := self.GetDatacenter() if err != nil { return err } - self.vms = make([]cloudprovider.ICloudVM, len(vms)) - for i := 0; i < len(vms); i += 1 { - self.vms[i] = NewVirtualMachine(self.manager, &vms[i], dc, self) + hostVms := self.getHostSystem().Vm + if len(hostVms) == 0 { + // log.Errorf("host VMs are nil!!!!!") + return nil } + + vms, err := dc.fetchVms(hostVms) + if err != nil { + return err + } + self.vms = vms return nil } @@ -497,3 +511,54 @@ func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error } return inics, nil } + +func (host *SHost) getLocalStorageCache() (*SDatastoreImageCache, error) { + if host.storageCache == nil { + sc, err := host.newLocalStorageCache() + if err != nil { + return nil, err + } + host.storageCache = sc + } + return host.storageCache, nil +} + +func (host *SHost) newLocalStorageCache() (*SDatastoreImageCache, error) { + ctx := context.Background() + + istorages, err := host.GetIStorages() + if err != nil { + return nil, err + } + var cacheDs *SDatastore + var maxDs *SDatastore + var maxCapacity int + for i := 0; i < len(istorages); i += 1 { + ds := istorages[i].(*SDatastore) + if !ds.isLocalVMFS() { + continue + } + _, err := ds.CheckFile(ctx, IMAGE_CACHE_DIR_NAME) + if err != nil { + if err != cloudprovider.ErrNotFound { + return nil, err + } + if maxCapacity < ds.GetCapacityMB() { + maxCapacity = ds.GetCapacityMB() + maxDs = ds + } + } else { + cacheDs = ds + break + } + } + if cacheDs == nil { + // if no existing image cache dir found, use the one with maximal capacilty + cacheDs = maxDs + } + + return &SDatastoreImageCache{ + datastore: cacheDs, + host: host, + }, nil +} diff --git a/pkg/util/esxi/image.go b/pkg/util/esxi/image.go new file mode 100644 index 0000000000..378c2d2c5b --- /dev/null +++ b/pkg/util/esxi/image.go @@ -0,0 +1,71 @@ +package esxi + +import ( + "context" + "path" + + "github.com/vmware/govmomi/object" + + "strings" + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SImage struct { + cache *SDatastoreImageCache + filename string +} + +func (self *SImage) getDatacenter() *object.Datacenter { + return self.cache.datastore.datacenter.getDcObj() +} + +func (self *SImage) getFullFilename() string { + return self.cache.datastore.getPathString(self.filename) +} + +func (self *SImage) GetId() string { + idstr := path.Base(self.filename) + if strings.HasSuffix(idstr, ".vmdk") { + idstr = idstr[:len(idstr)-5] + } + return strings.ToLower(idstr) +} + +func (self *SImage) GetName() string { + return path.Base(self.filename) +} + +func (self *SImage) GetGlobalId() string { + return self.GetId() +} + +func (self *SImage) GetStatus() string { + dm := object.NewVirtualDiskManager(self.cache.datastore.manager.client.Client) + ctx := context.Background() + _, err := dm.QueryVirtualDiskInfo(ctx, self.getFullFilename(), self.getDatacenter(), true) + if err != nil { + return "saving" + } + return "active" +} + +func (self *SImage) Refresh() error { + return nil +} + +func (self *SImage) IsEmulated() bool { + return false +} + +func (self *SImage) GetMetadata() *jsonutils.JSONDict { + return nil +} + +func (self *SImage) Delete(ctx context.Context) error { + return self.cache.datastore.DeleteVmdk(ctx, self.filename) +} + +func (self *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache { + return self.cache +} diff --git a/pkg/util/esxi/manager.go b/pkg/util/esxi/manager.go index 9b1f83da1b..e3ef4f4252 100644 --- a/pkg/util/esxi/manager.go +++ b/pkg/util/esxi/manager.go @@ -27,6 +27,8 @@ const ( ) type SESXiClient struct { + cloudprovider.SFakeOnPremiseRegion + providerId string providerName string host string @@ -180,6 +182,7 @@ func (cli *SESXiClient) references2Objects(refs []types.ManagedObjectReference, pc := property.DefaultCollector(cli.client.Client) err := pc.Retrieve(cli.context, refs, props, dst) if err != nil { + log.Errorf("pc.Retrieve fail %s", err) return err } return nil @@ -187,7 +190,12 @@ func (cli *SESXiClient) references2Objects(refs []types.ManagedObjectReference, func (cli *SESXiClient) reference2Object(ref types.ManagedObjectReference, props []string, dst interface{}) error { pc := property.DefaultCollector(cli.client.Client) - return pc.RetrieveOne(cli.context, ref, props, dst) + err := pc.RetrieveOne(cli.context, ref, props, dst) + if err != nil { + log.Errorf("pc.RetrieveOne fail %s", err) + return err + } + return nil } func (cli *SESXiClient) GetDatacenters() ([]*SDatacenter, error) { diff --git a/pkg/util/esxi/provider/provider.go b/pkg/util/esxi/provider/provider.go index de481a0188..f294f864f5 100644 --- a/pkg/util/esxi/provider/provider.go +++ b/pkg/util/esxi/provider/provider.go @@ -13,7 +13,6 @@ import ( ) type SESXiProviderFactory struct { - providerTable map[string]*SESXiProvider } func (self *SESXiProviderFactory) GetId() string { @@ -39,10 +38,6 @@ func parseHostPort(host string, defPort int) (string, int, error) { } func (self *SESXiProviderFactory) GetProvider(providerId, providerName, urlStr, account, secret string) (cloudprovider.ICloudProvider, error) { - provider, ok := self.providerTable[providerId] - if ok { - return provider, nil - } parts, err := url.Parse(urlStr) if err != nil { return nil, err @@ -56,14 +51,11 @@ func (self *SESXiProviderFactory) GetProvider(providerId, providerName, urlStr, if err != nil { return nil, err } - self.providerTable[providerId] = &SESXiProvider{client: client} - return self.providerTable[providerId], nil + return &SESXiProvider{client: client}, nil } func init() { - factory := SESXiProviderFactory{ - providerTable: make(map[string]*SESXiProvider), - } + factory := SESXiProviderFactory{} cloudprovider.RegisterFactory(&factory) } @@ -103,44 +95,10 @@ func (self *SESXiProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion return nil, cloudprovider.ErrNotSupported } -func (self *SESXiProvider) GetIHostById(id string) (cloudprovider.ICloudHost, error) { - host, err := self.client.FindHostByIp(id) - if err != nil { - return nil, err - } else { - return host, nil - } -} - -func (self *SESXiProvider) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { - return nil, cloudprovider.ErrNotSupported -} - -func (self *SESXiProvider) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return nil, cloudprovider.ErrNotSupported -} - -func (self *SESXiProvider) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - return nil, cloudprovider.ErrNotSupported -} - func (self *SESXiProvider) GetBalance() (float64, error) { return 0.0, nil } -func (self *SESXiProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { - dcs, err := self.client.GetDatacenters() - if err != nil { - return nil, err - } - - ihosts := make([]cloudprovider.ICloudHost, 0) - for i := 0; i < len(dcs); i += 1 { - dcIHosts, err := dcs[i].GetIHosts() - if err != nil { - return nil, err - } - ihosts = append(ihosts, dcIHosts...) - } - return ihosts, nil +func (self *SESXiProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) { + return self.client, nil } diff --git a/pkg/util/esxi/shell/store.go b/pkg/util/esxi/shell/store.go index ec5efa21e2..068931368c 100644 --- a/pkg/util/esxi/shell/store.go +++ b/pkg/util/esxi/shell/store.go @@ -2,12 +2,25 @@ package shell import ( "context" + "fmt" "os" "yunion.io/x/onecloud/pkg/util/esxi" "yunion.io/x/onecloud/pkg/util/printutils" "yunion.io/x/onecloud/pkg/util/shellutils" ) +func getDatastore(cli *esxi.SESXiClient, dcId string, dsId string) (*esxi.SDatastore, error) { + dc, err := cli.FindDatacenterByMoId(dcId) + if err != nil { + return nil, err + } + ds, err := dc.GetIStorageByMoId(dsId) + if err != nil { + return nil, err + } + return ds.(*esxi.SDatastore), nil +} + func init() { type DatastoreListOptions struct { DATACENTER string `help:"List datastores in datacenter"` @@ -30,11 +43,7 @@ func init() { DSID string `help:"Datastore ID""` } shellutils.R(&DatastoreShowOptions{}, "ds-show", "Show details of a datastore", func(cli *esxi.SESXiClient, args *DatastoreShowOptions) error { - dc, err := cli.FindDatacenterByMoId(args.DATACENTER) - if err != nil { - return err - } - ds, err := dc.GetIStorageByMoId(args.DSID) + ds, err := getDatastore(cli, args.DATACENTER, args.DSID) if err != nil { return err } @@ -42,22 +51,41 @@ func init() { return nil }) + shellutils.R(&DatastoreShowOptions{}, "ds-cache-show", "Show details of a datastore image cache", func(cli *esxi.SESXiClient, args *DatastoreShowOptions) error { + ds, err := getDatastore(cli, args.DATACENTER, args.DSID) + if err != nil { + return err + } + cache := ds.GetIStoragecache() + printObject(cache) + return nil + }) + + shellutils.R(&DatastoreShowOptions{}, "ds-cache-list", "Show image list of a datastore image cache", func(cli *esxi.SESXiClient, args *DatastoreShowOptions) error { + ds, err := getDatastore(cli, args.DATACENTER, args.DSID) + if err != nil { + return err + } + cache := ds.GetIStoragecache() + images, err := cache.GetIImages() + if err != nil { + return err + } + printList(images, []string{}) + return nil + }) + type DatastoreListDirOptions struct { DATACENTER string `help:"Datacenter"` DSID string `help:"Datastore ID""` DIR string `help:"directory"` } shellutils.R(&DatastoreListDirOptions{}, "ds-list-dir", "List directory of a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { - dc, err := cli.FindDatacenterByMoId(args.DATACENTER) - if err != nil { - return err - } - ds, err := dc.GetIStorageByMoId(args.DSID) + dsObj, err := getDatastore(cli, args.DATACENTER, args.DSID) if err != nil { return err } ctx := context.Background() - dsObj := ds.(*esxi.SDatastore) fileList, err := dsObj.ListDir(ctx, args.DIR) if err != nil { return err @@ -67,16 +95,11 @@ func init() { }) shellutils.R(&DatastoreListDirOptions{}, "ds-check-file", "Check file status in a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { - dc, err := cli.FindDatacenterByMoId(args.DATACENTER) - if err != nil { - return err - } - ds, err := dc.GetIStorageByMoId(args.DSID) + dsObj, err := getDatastore(cli, args.DATACENTER, args.DSID) if err != nil { return err } ctx := context.Background() - dsObj := ds.(*esxi.SDatastore) file, err := dsObj.CheckFile(ctx, args.DIR) if err != nil { return err @@ -86,20 +109,72 @@ func init() { }) shellutils.R(&DatastoreListDirOptions{}, "ds-delete-file", "Delete file in a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { - dc, err := cli.FindDatacenterByMoId(args.DATACENTER) - if err != nil { - return err - } - ds, err := dc.GetIStorageByMoId(args.DSID) + dsObj, err := getDatastore(cli, args.DATACENTER, args.DSID) if err != nil { return err } ctx := context.Background() - dsObj := ds.(*esxi.SDatastore) err = dsObj.Delete(ctx, args.DIR) if err != nil { return err } + fmt.Println("success") + return nil + }) + + shellutils.R(&DatastoreListDirOptions{}, "ds-check-vmdk", "Check vmdk file status in a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { + dsObj, err := getDatastore(cli, args.DATACENTER, args.DSID) + if err != nil { + return err + } + ctx := context.Background() + err = dsObj.CheckVmdk(ctx, args.DIR) + if err != nil { + return err + } + fmt.Println("valid") + return nil + }) + + shellutils.R(&DatastoreListDirOptions{}, "ds-delete-vmdk", "Delete vmdk file from a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { + dsObj, err := getDatastore(cli, args.DATACENTER, args.DSID) + if err != nil { + return err + } + ctx := context.Background() + err = dsObj.DeleteVmdk(ctx, args.DIR) + if err != nil { + return err + } + fmt.Println("success") + return nil + }) + + shellutils.R(&DatastoreListDirOptions{}, "ds-mkdir", "Delete vmdk directory from a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { + dsObj, err := getDatastore(cli, args.DATACENTER, args.DSID) + if err != nil { + return err + } + ctx := context.Background() + path, err := dsObj.MakeDir(ctx, args.DIR) + if err != nil { + return err + } + fmt.Println("Make dir success", path) + return nil + }) + + shellutils.R(&DatastoreListDirOptions{}, "ds-rmdir", "Remove vmdk directory from a datastore", func(cli *esxi.SESXiClient, args *DatastoreListDirOptions) error { + dsObj, err := getDatastore(cli, args.DATACENTER, args.DSID) + if err != nil { + return err + } + ctx := context.Background() + err = dsObj.RemoveDir(ctx, args.DIR) + if err != nil { + return err + } + fmt.Println("Remove dir success") return nil }) diff --git a/pkg/util/esxi/shell/virtualmachine.go b/pkg/util/esxi/shell/virtualmachine.go index 4d1ace464b..0313cad264 100644 --- a/pkg/util/esxi/shell/virtualmachine.go +++ b/pkg/util/esxi/shell/virtualmachine.go @@ -1,6 +1,8 @@ package shell import ( + "context" + "fmt" "yunion.io/x/onecloud/pkg/util/esxi" "yunion.io/x/onecloud/pkg/util/printutils" "yunion.io/x/onecloud/pkg/util/shellutils" @@ -74,6 +76,33 @@ func init() { return nil }) + type VirtualMachineDiskResizeOptions struct { + HOSTIP string `help:"host ip"` + VMID string `help:"virtual machine UUID"` + DISKIDX int `help:"disk index"` + SIZEGB int64 `help:"new size of disk"` + } + shellutils.R(&VirtualMachineDiskResizeOptions{}, "vm-disk-resize", "Resize a vm disk", func(cli *esxi.SESXiClient, args *VirtualMachineDiskResizeOptions) error { + host, err := cli.FindHostByIp(args.HOSTIP) + if err != nil { + return err + } + vm, err := host.GetIVMById(args.VMID) + if err != nil { + return err + } + vmdisks, err := vm.GetIDisks() + if err != nil { + return err + } + if args.DISKIDX < 0 || args.DISKIDX >= len(vmdisks) { + return fmt.Errorf("Out of index: %d", args.DISKIDX) + } + disk := vmdisks[args.DISKIDX] + ctx := context.Background() + return disk.Resize(ctx, args.SIZEGB*1024) + }) + shellutils.R(&VirtualMachineShowOptions{}, "vm-vnc", "Show vm VNC details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error { host, err := cli.FindHostByIp(args.HOSTIP) if err != nil { diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 0b0b2576d2..34c786b937 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -5,6 +5,7 @@ import ( "context" "fmt" + "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/types" "io" "io/ioutil" @@ -22,12 +23,16 @@ import ( "yunion.io/x/onecloud/pkg/util/vmdkutils" ) -var DATASTORE_PROPS = []string{"name", "parent", "info", "summary", "host"} +var DATASTORE_PROPS = []string{"name", "parent", "info", "summary", "host", "vm"} type SDatastore struct { SManagedObject + // vms []cloudprovider.ICloudVM + ihosts []cloudprovider.ICloudHost + + storageCache *SDatastoreImageCache } func NewDatastore(manager *SESXiClient, ds *mo.Datastore, dc *SDatacenter) *SDatastore { @@ -76,7 +81,16 @@ func (self *SDatastore) GetStatus() string { } func (self *SDatastore) Refresh() error { - return cloudprovider.ErrNotImplemented + base := self.SManagedObject + var moObj mo.Datastore + err := self.manager.reference2Object(self.object.Reference(), DATASTORE_PROPS, &moObj) + if err != nil { + return err + } + base.object = &moObj + *self = SDatastore{} + self.SManagedObject = base + return nil } func (self *SDatastore) IsEmulated() bool { @@ -179,19 +193,80 @@ func (self *SDatastore) getLocalHost() (cloudprovider.ICloudHost, error) { } func (self *SDatastore) GetIStoragecache() cloudprovider.ICloudStoragecache { - return nil + if self.isLocalVMFS() { + ihost, err := self.getLocalHost() + if err != nil { + log.Errorf("GetIStoragecache getLocalHost fail %s", err) + return nil + } + host := ihost.(*SHost) + sc, err := host.getLocalStorageCache() + if err != nil { + log.Errorf("GetIStoragecache getLocalStorageCache fail %s", err) + return nil + } + return sc + } else { + return self.getStorageCache() + } +} + +func (self *SDatastore) getStorageCache() *SDatastoreImageCache { + if self.storageCache == nil { + self.storageCache = &SDatastoreImageCache{ + datastore: self, + } + } + return self.storageCache } func (self *SDatastore) GetIZone() cloudprovider.ICloudZone { return nil } -func (self *SDatastore) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { - return nil, cloudprovider.ErrNotImplemented +func (self *SDatastore) getVMs() ([]cloudprovider.ICloudVM, error) { + dc, err := self.GetDatacenter() + if err != nil { + log.Errorf("SDatastore GetDatacenter fail %s", err) + return nil, err + } + vms := self.getDatastore().Vm + if len(vms) == 0 { + return nil, nil + } + return dc.fetchVms(vms) +} + +func (self *SDatastore) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { + vms, err := self.getVMs() + if err != nil { + log.Errorf("self.getVMs fail %s", err) + return nil, err + } + for i := 0; i < len(vms); i += 1 { + vm := vms[i].(*SVirtualMachine) + disk, err := vm.GetIDiskById(idStr) + if err == nil { + return disk, nil + } + } + return nil, cloudprovider.ErrNotFound } func (self *SDatastore) GetIDisks() ([]cloudprovider.ICloudDisk, error) { - return nil, cloudprovider.ErrNotImplemented + vms, err := self.getVMs() + if err != nil { + return nil, err + } + allDisks := make([]cloudprovider.ICloudDisk, 0) + for i := 0; i < len(vms); i += 1 { + disks, err := vms[i].GetIDisks() + if err != nil { + return nil, err + } + allDisks = append(allDisks, disks...) + } + return allDisks, nil } func (self *SDatastore) isLocalVMFS() bool { @@ -259,6 +334,10 @@ func (self *SDatastore) GetUrl() string { return self.getDatastore().Info.GetDatastoreInfo().Url } +func (self *SDatastore) GetMountPoint() string { + return self.GetUrl() +} + func (self *SDatastore) cleanPath(remotePath string) string { dsName := fmt.Sprintf("[%s]", self.SManagedObject.GetName()) dsUrl := self.GetUrl() @@ -298,6 +377,11 @@ func (self *SDatastore) getPathString(path string) string { return fmt.Sprintf("[%s] %s", self.SManagedObject.GetName(), path) } +func (self *SDatastore) getFullPath(remotePath string) string { + remotePath = self.cleanPath(remotePath) + return path.Join(self.GetUrl(), remotePath) +} + func (self *SDatastore) CreateIDisk(name string, sizeGb int, desc string) (cloudprovider.ICloudDisk, error) { return nil, cloudprovider.ErrNotImplemented } @@ -396,6 +480,9 @@ func (self *SDatastore) CheckFile(ctx context.Context, remotePath string) (*SDat err = self.manager.client.Do(ctx, req, func(resp *http.Response) error { if resp.StatusCode >= 400 { + if resp.StatusCode == 404 { + return cloudprovider.ErrNotFound + } return fmt.Errorf("%s", resp.Status) } sizeStr := resp.Header.Get("Content-Length") @@ -502,6 +589,13 @@ func (self *SDatastore) Delete(ctx context.Context, remotePath string) error { } func (self *SDatastore) DeleteVmdk(ctx context.Context, remotePath string) error { + info, err := self.CheckFile(ctx, remotePath) + if err != nil { + return err + } + if info.Size > 4096 { + return fmt.Errorf("not a valid vmdk file") + } vmdkContent, err := self.FileGetContent(ctx, remotePath) if err != nil { return err @@ -515,10 +609,59 @@ func (self *SDatastore) DeleteVmdk(ctx context.Context, remotePath string) error return err } if len(vmdkInfo.ExtentFile) > 0 { - err = self.Delete(ctx, path.Join(remotePath, vmdkInfo.ExtentFile)) + err = self.Delete(ctx, path.Join(path.Dir(remotePath), vmdkInfo.ExtentFile)) if err != nil { return err } } return nil } + +func (self *SDatastore) CheckVmdk(ctx context.Context, remotePath string) error { + dm := object.NewVirtualDiskManager(self.manager.client.Client) + defer dm.Destroy(ctx) + + dc, err := self.GetDatacenter() + if err != nil { + return err + } + + dcObj := dc.getObjectDatacenter() + + infoList, err := dm.QueryVirtualDiskInfo(ctx, self.getPathString(remotePath), dcObj, true) + if err != nil { + return err + } + + log.Debugf("%#v", infoList) + return nil +} + +func (self *SDatastore) getDatastoreObj() *object.Datastore { + return object.NewDatastore(self.manager.client.Client, self.getDatastore().Self) +} + +func (self *SDatastore) MakeDir(ctx context.Context, remotePath string) (string, error) { + dnm := object.NewDatastoreNamespaceManager(self.manager.client.Client) + + remotePath = self.cleanPath(remotePath) + + objDS := self.getDatastoreObj() + + return dnm.CreateDirectory(ctx, objDS, remotePath, "") +} + +func (self *SDatastore) RemoveDir(ctx context.Context, remotePath string) error { + dnm := object.NewDatastoreNamespaceManager(self.manager.client.Client) + + remotePath = self.getFullPath(remotePath) + + dc, err := self.GetDatacenter() + if err != nil { + return err + } + + dcObj := dc.getObjectDatacenter() + + return dnm.DeleteDirectory(ctx, dcObj, remotePath) +} diff --git a/pkg/util/esxi/storagecache.go b/pkg/util/esxi/storagecache.go new file mode 100644 index 0000000000..a2268b27a9 --- /dev/null +++ b/pkg/util/esxi/storagecache.go @@ -0,0 +1,132 @@ +package esxi + +import ( + "context" + "fmt" + "path" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" +) + +const ( + IMAGE_CACHE_DIR_NAME = "image_cache" +) + +type SDatastoreImageCache struct { + datastore *SDatastore + host *SHost +} + +func (self *SDatastoreImageCache) GetId() string { + if self.host != nil { + return self.host.GetGlobalId() + } else { + return self.datastore.GetGlobalId() + } +} + +func (self *SDatastoreImageCache) GetName() string { + if self.host != nil { + return fmt.Sprintf("storage-cache-%s", self.host.GetName()) + } else { + return fmt.Sprintf("storage-cache-%s", self.datastore.GetName()) + } +} + +func (self *SDatastoreImageCache) GetGlobalId() string { + return self.GetId() +} + +func (self *SDatastoreImageCache) GetStatus() string { + return "available" +} + +func (self *SDatastoreImageCache) Refresh() error { + return nil +} + +func (self *SDatastoreImageCache) IsEmulated() bool { + return false +} + +func (self *SDatastoreImageCache) GetMetadata() *jsonutils.JSONDict { + return nil +} + +func (self *SDatastoreImageCache) GetPath() string { + return path.Join(self.datastore.GetMountPoint(), IMAGE_CACHE_DIR_NAME) +} + +func (self *SDatastoreImageCache) GetIImages() ([]cloudprovider.ICloudImage, error) { + ctx := context.Background() + + files, err := self.datastore.ListDir(ctx, IMAGE_CACHE_DIR_NAME) + if err != nil { + log.Errorf("GetIImages ListDir fail %s", err) + return nil, err + } + + ret := make([]cloudprovider.ICloudImage, 0) + + validFilenames := make(map[string]bool) + + for i := 0; i < len(files); i += 1 { + filename := path.Join(IMAGE_CACHE_DIR_NAME, files[i].Name) + if err := self.datastore.CheckVmdk(ctx, filename); err != nil { + continue + } + image := SImage{ + cache: self, + filename: filename, + } + ret = append(ret, &image) + vmdkName := files[i].Name + vmdkExtName := fmt.Sprintf("%s-flat.vmdk", vmdkName[:len(vmdkName)-5]) + validFilenames[vmdkName] = true + validFilenames[vmdkExtName] = true + } + + log.Debugf("storage cache contains %#v", validFilenames) + // cleanup storage cache!!! + for i := 0; i < len(files); i += 1 { + if _, ok := validFilenames[files[i].Name]; !ok { + log.Debugf("delete invalid vmdk file %s!!!", files[i].Name) + self.datastore.Delete(ctx, path.Join(IMAGE_CACHE_DIR_NAME, files[i].Name)) + } + } + + return ret, nil +} + +func (self *SDatastoreImageCache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { + images, err := self.GetIImages() + if err != nil { + return nil, err + } + for i := 0; i < len(images); i += 1 { + if images[i].GetGlobalId() == extId { + return images[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (self *SDatastoreImageCache) GetManagerId() string { + return self.datastore.manager.providerId +} + +func (self *SDatastoreImageCache) CreateIImage(snapshotId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (self *SDatastoreImageCache) DownloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (self *SDatastoreImageCache) UploadImage(userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, extId string, isForce bool) (string, error) { + return "", cloudprovider.ErrNotImplemented +} diff --git a/pkg/util/esxi/utils_test.go b/pkg/util/esxi/utils_test.go new file mode 100644 index 0000000000..93d0c83405 --- /dev/null +++ b/pkg/util/esxi/utils_test.go @@ -0,0 +1,31 @@ +package esxi + +import "testing" + +func TestInitValue(t *testing.T) { + type testStruct struct { + member1 int + member2 string + Member3 int + Member4 string + } + + dst := testStruct{} + + dst.member1 = 1 + dst.member2 = "2" + dst.Member3 = 3 + dst.Member4 = "4" + + pDst := &dst + t.Logf("%p %#v", pDst, pDst) + + *pDst = testStruct{} + + t.Logf("%p %#v", pDst, pDst) + + new := testStruct{} + if dst != new { + t.Errorf("dst != new") + } +} diff --git a/pkg/util/esxi/vdisk.go b/pkg/util/esxi/vdisk.go index 8334b7d22b..0d0c63b97b 100644 --- a/pkg/util/esxi/vdisk.go +++ b/pkg/util/esxi/vdisk.go @@ -1,17 +1,17 @@ package esxi import ( + "context" "fmt" "path" + "strings" + "time" "github.com/vmware/govmomi/vim25/types" "yunion.io/x/jsonutils" "yunion.io/x/log" - "context" - "strings" - "time" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -90,8 +90,14 @@ func (disk *SVirtualDisk) GetDiskSizeMB() int { return int(capa / 1024 / 1024) } -func (disk *SVirtualDisk) getFilename() string { - return disk.getBackingInfo().FileName +func (disk *SVirtualDisk) GetAccessPath() string { + istore, err := disk.GetIStorage() + if err != nil { + log.Errorf("disk.GetIStorage fail %s", err) + return "" + } + ds := istore.(*SDatastore) + return ds.getFullPath(disk.getBackingInfo().FileName) } func (disk *SVirtualDisk) GetDiskFormat() string { @@ -127,7 +133,6 @@ func (disk *SVirtualDisk) GetTemplateId() string { func (disk *SVirtualDisk) GetDiskType() string { backing := disk.getBackingInfo() if backing.Parent != nil { - // log.Infof("%s", backing.Parent.FileName) return models.DISK_TYPE_SYS } return models.DISK_TYPE_DATA @@ -176,10 +181,11 @@ func (disk *SVirtualDisk) GetMountpoint() string { func (disk *SVirtualDisk) Delete(ctx context.Context) error { istorage, err := disk.GetIStorage() if err != nil { + log.Errorf("disk.GetIStorage() fail %s", err) return err } ds := istorage.(*SDatastore) - return ds.DeleteVmdk(ctx, disk.getFilename()) + return ds.DeleteVmdk(ctx, disk.getBackingInfo().FileName) } func (disk *SVirtualDisk) CreateISnapshot(ctx context.Context, name string, desc string) (cloudprovider.ICloudSnapshot, error) { @@ -194,8 +200,33 @@ func (disk *SVirtualDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error return nil, cloudprovider.ErrNotImplemented } -func (disk *SVirtualDisk) Resize(ctx context.Context, newSize int64) error { - return cloudprovider.ErrNotImplemented +func (disk *SVirtualDisk) Resize(ctx context.Context, newSizeMb int64) error { + ndisk := disk.getVirtualDisk() + ndisk.CapacityInKB = newSizeMb * 1024 + + devSpec := types.VirtualDeviceConfigSpec{} + devSpec.Device = ndisk + devSpec.Operation = types.VirtualDeviceConfigSpecOperationEdit + + spec := types.VirtualMachineConfigSpec{} + spec.DeviceChange = []types.BaseVirtualDeviceConfigSpec{&devSpec} + + vm := disk.vm.getVmObj() + + task, err := vm.Reconfigure(ctx, spec) + + if err != nil { + log.Errorf("vm.Reconfigure fail %s", err) + return err + } + + err = task.Wait(ctx) + if err != nil { + log.Errorf("task.Wait fail %s", err) + return err + } + + return err } func (disk *SVirtualDisk) Reset(ctx context.Context, snapshotId string) error { diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 2000bc0890..e25b4547d3 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -3,6 +3,7 @@ package esxi import ( "context" "fmt" + "reflect" "time" "github.com/vmware/govmomi/object" @@ -12,12 +13,13 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/secrules" - "reflect" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/pkg/util/netutils" - "yunion.io/yunioncloud/pkg/util/regutils" + "yunion.io/x/pkg/util/regutils" + + "strings" ) var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "runtime", "summary", "config", "guest"} @@ -32,12 +34,10 @@ type SVirtualMachine struct { devs map[int32]SVirtualDevice guestIps map[string]string - - host *SHost } -func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacenter, host *SHost) *SVirtualMachine { - svm := &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc), host: host} +func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacenter) *SVirtualMachine { + svm := &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc)} svm.fetchHardwareInfo() return svm } @@ -77,6 +77,16 @@ func (self *SVirtualMachine) GetStatus() string { } func (self *SVirtualMachine) Refresh() error { + base := self.SManagedObject + var moObj mo.VirtualMachine + err := self.manager.reference2Object(self.object.Reference(), VIRTUAL_MACHINE_PROPS, &moObj) + if err != nil { + return err + } + base.object = &moObj + *self = SVirtualMachine{} + self.SManagedObject = base + self.fetchHardwareInfo() return nil } @@ -119,16 +129,16 @@ func (self *SVirtualMachine) GetCreateTime() time.Time { } func (self *SVirtualMachine) GetIHost() cloudprovider.ICloudHost { - // moVM := self.getVirtualMachine() - // log.Debugf("%#v", moVM.Parent) - me := self.findInParents("HostSystem") - if me == nil { - log.Errorf("fail to find vm host??? %s", self.GetName()) - return self.host - } - ihost, err := self.manager.FindHostByMoId(me.Self.Value) + vm := self.getVmObj() + + hostsys, err := vm.HostSystem(self.manager.context) if err != nil { - log.Errorf("fail to find host %s for vm %s???", me.Self.Value, self.GetName()) + log.Errorf("fail to find host system for vm %s", err) + return nil + } + ihost, err := self.manager.FindHostByMoId(moRefId(hostsys.Reference())) + if err != nil { + log.Errorf("fail to find host %s for vm %s???", hostsys.Name(), self.GetName()) return nil } return ihost @@ -142,6 +152,15 @@ func (self *SVirtualMachine) GetIDisks() ([]cloudprovider.ICloudDisk, error) { return idisks, nil } +func (self *SVirtualMachine) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { + for i := 0; i < len(self.vdisks); i += 1 { + if self.vdisks[i].GetGlobalId() == idStr { + return &self.vdisks[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + func (self *SVirtualMachine) GetINics() ([]cloudprovider.ICloudNic, error) { inics := make([]cloudprovider.ICloudNic, len(self.vnics)) for i := 0; i < len(self.vnics); i += 1 { @@ -199,16 +218,32 @@ func (self *SVirtualMachine) GetGuestToolsStatus() string { return string(moVM.Guest.ToolsStatus) } +func (self *SVirtualMachine) isToolsOk() bool { + switch self.getVirtualMachine().Guest.ToolsStatus { + case types.VirtualMachineToolsStatusToolsNotInstalled: + return false + case types.VirtualMachineToolsStatusToolsNotRunning: + return false + } + return true +} + func (self *SVirtualMachine) GetGuestToolsRunningStatus() string { moVM := self.getVirtualMachine() return string(moVM.Guest.ToolsRunningStatus) } func (self *SVirtualMachine) GetOSType() string { + if osInfo, ok := GuestOsInfo[self.GetGuestId()]; ok { + return string(osInfo.OsType) + } return "" } func (self *SVirtualMachine) GetOSName() string { + if osInfo, ok := GuestOsInfo[self.GetGuestId()]; ok { + return string(osInfo.OsDistribution) + } return "" } @@ -236,12 +271,23 @@ func (self *SVirtualMachine) getVmObj() *object.VirtualMachine { return object.NewVirtualMachine(self.manager.client.Client, self.getVirtualMachine().Self) } +// ideopotent start func (self *SVirtualMachine) StartVM(ctx context.Context) error { + if self.GetStatus() == models.VM_RUNNING { + return nil + } + return self.startVM(ctx) +} + +func (self *SVirtualMachine) startVM(ctx context.Context) error { err := self.makeNicsStartConnected(ctx) if err != nil { return err } - task, err := self.getVmObj().PowerOn(ctx) + + vm := self.getVmObj() + + task, err := vm.PowerOn(ctx) if err != nil { return err } @@ -254,7 +300,10 @@ func (self *SVirtualMachine) makeNicsStartConnected(ctx context.Context) error { for i := 0; i < len(self.vnics); i += 1 { spec.DeviceChange[i] = makeNicStartConnected(&self.vnics[i]) } - task, err := self.getVmObj().Reconfigure(ctx, spec) + + vm := self.getVmObj() + + task, err := vm.Reconfigure(ctx, spec) if err != nil { return err } @@ -271,18 +320,44 @@ func makeNicStartConnected(nic *SVirtualNIC) *types.VirtualDeviceConfigSpec { } func (self *SVirtualMachine) StopVM(ctx context.Context, isForce bool) error { - task, err := self.getVmObj().PowerOff(ctx) + if self.GetStatus() == models.VM_READY { + return nil + } + if !isForce && self.isToolsOk() { + return self.shutdownVM(ctx) + } else { + return self.poweroffVM(ctx) + } +} + +func (self *SVirtualMachine) poweroffVM(ctx context.Context) error { + vm := self.getVmObj() + + task, err := vm.PowerOff(ctx) if err != nil { return err } return task.Wait(ctx) } -func (self *SVirtualMachine) doDelete(ctx context.Context) error { - task, err := self.getVmObj().Destroy(ctx) +func (self *SVirtualMachine) shutdownVM(ctx context.Context) error { + vm := self.getVmObj() + + err := vm.ShutdownGuest(ctx) if err != nil { return err } + return err +} + +func (self *SVirtualMachine) doDelete(ctx context.Context) error { + vm := self.getVmObj() + + task, err := vm.Destroy(ctx) + if err != nil { + log.Errorf("vm.Destroy(ctx) fail %s", err) + return err + } return task.Wait(ctx) } @@ -290,6 +365,7 @@ func (self *SVirtualMachine) DeleteVM(ctx context.Context) error { for i := 0; i < len(self.vdisks); i += 1 { err := self.doDetachAndDeleteDisk(ctx, &self.vdisks[i]) if err != nil { + log.Errorf("self.doDetachAndDeleteDisk(ctx, &self.vdisks[i]) fail %s", err) return err } } @@ -308,13 +384,17 @@ func (self *SVirtualMachine) doDetachDisk(ctx context.Context, vdisk *SVirtualDi spec := types.VirtualMachineConfigSpec{} spec.DeviceChange = []types.BaseVirtualDeviceConfigSpec{&removeSpec} - task, err := self.getVmObj().Reconfigure(ctx, spec) + vm := self.getVmObj() + + task, err := vm.Reconfigure(ctx, spec) if err != nil { + log.Errorf("vm.Reconfigure fail %s", err) return err } err = task.Wait(ctx) if err != nil { + log.Errorf("task.Wait(ctx) fail %s", err) return err } @@ -382,8 +462,48 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { return ret, nil } -func (dc *SVirtualMachine) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { - return cloudprovider.ErrNotImplemented +func (self *SVirtualMachine) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { + return self.doChangeConfig(ctx, int32(ncpu), int64(vmem), "", "") +} + +func (self *SVirtualMachine) GetVersion() string { + return self.getVirtualMachine().Config.Version +} + +func (self *SVirtualMachine) doChangeConfig(ctx context.Context, ncpu int32, vmemMB int64, guestId string, version string) error { + changed := false + configSpec := types.VirtualMachineConfigSpec{} + if int8(ncpu) != self.GetVcpuCount() { + configSpec.NumCPUs = ncpu + changed = true + } + if int(vmemMB) != self.GetVmemSizeMB() { + configSpec.MemoryMB = vmemMB + changed = true + } + if len(guestId) > 0 && guestId != self.GetGuestId() { + configSpec.GuestId = guestId + changed = true + } + if len(version) > 0 && version != self.GetVersion() { + configSpec.Version = version + changed = true + } + if !changed { + return nil + } + + vm := self.getVmObj() + + task, err := vm.Reconfigure(ctx, configSpec) + if err != nil { + return err + } + err = task.Wait(ctx) + if err != nil { + return err + } + return self.Refresh() } func (self *SVirtualMachine) GetBillingType() string { @@ -466,3 +586,100 @@ func (self *SVirtualMachine) GetIps() []string { func (self *SVirtualMachine) GetVGADevice() string { return fmt.Sprintf("%s", self.vga.String()) } + +var ( + driverTable = map[string][]string{ + "sata": {"ahci"}, + "scsi": {"lsilogic", "lsilogicsas", "buslogic"}, + "pvscsi": {"parascsi"}, + "ide": {"ide"}, + } +) + +func (self *SVirtualMachine) getDevsByDriver(driver string) []SVirtualDevice { + devs := make([]SVirtualDevice, 0) + for _, drv := range self.devs { + if strings.HasSuffix(drv.GetDriver(), fmt.Sprintf("%scontroller", driver)) { + devs = append(devs, drv) + } + } + return devs +} + +func minDevKey(devs []SVirtualDevice) int32 { + var minKey int32 = -1 + for i := 0; i < len(devs); i += 1 { + if minKey < 0 || minKey > devs[i].getKey() { + minKey = devs[i].getKey() + } + } + return minKey +} + +func minDiskKey(devs []SVirtualDisk) int32 { + var minKey int32 = -1 + for i := 0; i < len(devs); i += 1 { + if minKey < 0 || minKey > devs[i].getKey() { + minKey = devs[i].getKey() + } + } + return minKey +} + +func (self *SVirtualMachine) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { + aliasDrivers, ok := driverTable[driver] + if !ok { + return fmt.Errorf("Unsupported disk driver %s", driver) + } + var devs []SVirtualDevice + for _, alias := range aliasDrivers { + devs = self.getDevsByDriver(alias) + if len(devs) > 0 { + break + } + } + if len(devs) == 0 { + return fmt.Errorf("Driver %s not found", driver) + } + ctlKey := minDevKey(devs) + sameDisks := make([]SVirtualDisk, 0) + for i := 0; i < len(self.vdisks); i += 1 { + if self.vdisks[i].GetDriver() == driver { + sameDisks = append(sameDisks, self.vdisks[i]) + } + } + var diskKey int32 = 2000 + if len(sameDisks) == 0 { + diskKey = minDiskKey(sameDisks) + } + index := len(sameDisks) + if driver == "ide" { + ctlKey += int32(index / 2) + } + devSpec := NewDiskDev(int64(sizeMb), "", uuid, int32(index), diskKey, ctlKey) + spec := addDevSpec(devSpec) + spec.FileOperation = types.VirtualDeviceConfigSpecFileOperationCreate + configSpec := types.VirtualMachineConfigSpec{} + configSpec.DeviceChange = []types.BaseVirtualDeviceConfigSpec{spec} + + vmObj := self.getVmObj() + + task, err := vmObj.Reconfigure(ctx, configSpec) + if err != nil { + return err + } + err = task.Wait(ctx) + if err != nil { + return err + } + oldDiskCnt := len(self.vdisks) + maxTries := 60 + for tried := 0; tried < maxTries; tried += 1 { + time.Sleep(time.Second) + self.Refresh() + if len(self.vdisks) > oldDiskCnt { + return nil + } + } + return cloudprovider.ErrTimeout +} diff --git a/pkg/util/qcloud/disk.go b/pkg/util/qcloud/disk.go index 8705edcdfe..1f3636074f 100644 --- a/pkg/util/qcloud/disk.go +++ b/pkg/util/qcloud/disk.go @@ -176,8 +176,8 @@ func (self *SRegion) ResizeDisk(ctx context.Context, diskId string, sizeGb int64 } } -func (self *SDisk) Resize(ctx context.Context, size int64) error { - return self.storage.zone.region.ResizeDisk(ctx, self.DiskId, size) +func (self *SDisk) Resize(ctx context.Context, sizeMb int64) error { + return self.storage.zone.region.ResizeDisk(ctx, self.DiskId, sizeMb/1024) } func (self *SDisk) GetName() string { @@ -374,3 +374,7 @@ func (self *SRegion) CreateDisk(zoneId string, category string, name string, siz } return diskIDSet[0], nil } + +func (disk *SDisk) GetAccessPath() string { + return "" +} diff --git a/pkg/util/qcloud/image.go b/pkg/util/qcloud/image.go index 2d481ab146..e368758ae9 100644 --- a/pkg/util/qcloud/image.go +++ b/pkg/util/qcloud/image.go @@ -5,6 +5,7 @@ import ( "strconv" "time" + "context" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -104,7 +105,7 @@ func (self *SImage) GetGlobalId() string { return fmt.Sprintf("%s-%s") } -func (self *SImage) Delete() error { +func (self *SImage) Delete(ctx context.Context) error { return self.storageCache.region.DeleteImage(self.ImageId) } diff --git a/pkg/util/qcloud/instance.go b/pkg/util/qcloud/instance.go index ba0103bffa..502dd52c35 100644 --- a/pkg/util/qcloud/instance.go +++ b/pkg/util/qcloud/instance.go @@ -400,7 +400,7 @@ func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd s return instance.SystemDisk.DiskId, nil } -func (self *SInstance) ChangeConfig(ctx context.Context, instanceId string, ncpu int, vmem int) error { +func (self *SInstance) ChangeConfig(ctx context.Context, ncpu int, vmem int) error { return self.host.zone.region.ChangeVMConfig(self.Placement.Zone, self.InstanceId, ncpu, vmem, nil) } @@ -735,3 +735,7 @@ func (self *SInstance) GetExpiredAt() time.Time { func (self *SInstance) UpdateUserData(userData string) error { return cloudprovider.ErrNotSupported } + +func (self *SInstance) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { + return cloudprovider.ErrNotSupported +} diff --git a/pkg/util/qcloud/localdisk.go b/pkg/util/qcloud/localdisk.go index ab100b7a69..5d1e724d0d 100644 --- a/pkg/util/qcloud/localdisk.go +++ b/pkg/util/qcloud/localdisk.go @@ -127,3 +127,7 @@ func (self *SLocalDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { func (self *SLocalDisk) Resize(ctx context.Context, size int64) error { return cloudprovider.ErrNotSupported } + +func (disk *SLocalDisk) GetAccessPath() string { + return "" +} diff --git a/pkg/util/qcloud/localstorage.go b/pkg/util/qcloud/localstorage.go index 38d57186c1..5d12e21564 100644 --- a/pkg/util/qcloud/localstorage.go +++ b/pkg/util/qcloud/localstorage.go @@ -93,6 +93,10 @@ func (self *SLocalStorage) CreateIDisk(name string, sizeGb int, desc string) (cl return nil, cloudprovider.ErrNotSupported } -func (self *SLocalStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { +func (self *SLocalStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { return &SLocalDisk{storage: self, DiskId: idStr}, nil } + +func (self *SLocalStorage) GetMountPoint() string { + return "" +} diff --git a/pkg/util/qcloud/provider/provider.go b/pkg/util/qcloud/provider/provider.go index 9f34d9123a..932b441855 100644 --- a/pkg/util/qcloud/provider/provider.go +++ b/pkg/util/qcloud/provider/provider.go @@ -67,26 +67,10 @@ func (self *SQcloudProvider) GetIRegionById(id string) (cloudprovider.ICloudRegi return self.client.GetIRegionById(id) } -func (self *SQcloudProvider) GetIHostById(id string) (cloudprovider.ICloudHost, error) { - return self.client.GetIHostById(id) -} - -func (self *SQcloudProvider) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { - return self.client.GetIVpcById(id) -} - -func (self *SQcloudProvider) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return self.client.GetIStorageById(id) -} - -func (self *SQcloudProvider) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - return self.client.GetIStoragecacheById(id) -} - func (self *SQcloudProvider) GetBalance() (float64, error) { return 0.0, nil } -func (self *SQcloudProvider) GetOnPremiseIHosts() ([]cloudprovider.ICloudHost, error) { +func (self *SQcloudProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/util/qcloud/qcloud.go b/pkg/util/qcloud/qcloud.go index 02fb2fd061..492ab5ab6d 100644 --- a/pkg/util/qcloud/qcloud.go +++ b/pkg/util/qcloud/qcloud.go @@ -250,18 +250,6 @@ func (client *SQcloudClient) GetIStorageById(id string) (cloudprovider.ICloudSto return nil, cloudprovider.ErrNotFound } -func (client *SQcloudClient) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { - for i := 0; i < len(client.iregions); i++ { - ihost, err := client.iregions[i].GetIStoragecacheById(id) - if err == nil { - return ihost, nil - } else if err != cloudprovider.ErrNotFound { - return nil, err - } - } - return nil, cloudprovider.ErrNotFound -} - type SAccountBalance struct { AvailableAmount float64 AvailableCashAmount float64 diff --git a/pkg/util/qcloud/region.go b/pkg/util/qcloud/region.go index 2c74f880b5..aac721a209 100644 --- a/pkg/util/qcloud/region.go +++ b/pkg/util/qcloud/region.go @@ -156,6 +156,40 @@ func (self *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, er return nil, cloudprovider.ErrNotFound } +func (self *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) { + iHosts := make([]cloudprovider.ICloudHost, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneHost, err := izones[i].GetIHosts() + if err != nil { + return nil, err + } + iHosts = append(iHosts, iZoneHost...) + } + return iHosts, nil +} + +func (self *SRegion) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + iStores := make([]cloudprovider.ICloudStorage, 0) + + izones, err := self.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneStores, err := izones[i].GetIStorages() + if err != nil { + return nil, err + } + iStores = append(iStores, iZoneStores...) + } + return iStores, nil +} + func (self *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { storageCache := self.getStoragecache() if storageCache.GetGlobalId() == id { diff --git a/pkg/util/qcloud/storage.go b/pkg/util/qcloud/storage.go index a9ebca1bc4..065210f5df 100644 --- a/pkg/util/qcloud/storage.go +++ b/pkg/util/qcloud/storage.go @@ -121,7 +121,7 @@ func (self *SStorage) CreateIDisk(name string, sizeGb int, desc string) (cloudpr return nil, cloudprovider.ErrNotFound } -func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { +func (self *SStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { disk, err := self.zone.region.GetDisk(idStr) if err != nil { return nil, err @@ -129,3 +129,7 @@ func (self *SStorage) GetIDisk(idStr string) (cloudprovider.ICloudDisk, error) { disk.storage = self return disk, nil } + +func (self *SStorage) GetMountPoint() string { + return "" +} diff --git a/pkg/util/qcloud/storagecache.go b/pkg/util/qcloud/storagecache.go index f51281d605..c1d4e64571 100644 --- a/pkg/util/qcloud/storagecache.go +++ b/pkg/util/qcloud/storagecache.go @@ -109,6 +109,22 @@ func (self *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { return self.iimages, nil } +func (self *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { + parts, _, err := self.region.GetImages("", "PRIVATE_IMAGE", []string{extId}, "", 0, 1) + if err != nil { + return nil, err + } + if len(parts) == 0 { + return nil, cloudprovider.ErrNotFound + } + parts[1].storageCache = self + return &parts[0], nil +} + +func (self *SStoragecache) GetPath() string { + return "" +} + func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, extId string, isForce bool) (string, error) { if len(extId) > 0 { log.Debugf("UploadImage: Image external ID exists %s", extId) From df4adb6a6ffcf4168e4255ef97d90700664e05e8 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sun, 25 Nov 2018 09:34:18 +0800 Subject: [PATCH 03/10] remote sysutils --- pkg/compute/guestdrivers/qcloud.go | 1 - pkg/compute/models/disks.go | 3 +-- pkg/compute/models/guests.go | 3 +-- pkg/compute/models/hosts.go | 3 +-- pkg/compute/models/storages.go | 5 ++--- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index baa50e286c..e1dc7c1536 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -9,7 +9,6 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index c1125abc29..4a623113a4 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -15,7 +15,6 @@ import ( "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -1026,7 +1025,7 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info diskConfig.Mountpoint = p } else if p == "autoextend" { diskConfig.Size = -1 - } else if utils.IsInStringArray(p, sysutils.STORAGE_TYPES) { + } else if utils.IsInStringArray(p, STORAGE_TYPES) { diskConfig.Backend = p } else if len(p) > 0 { if userCred == nil { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 246b9405ca..660acf7093 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -20,7 +20,6 @@ import ( "yunion.io/x/pkg/util/osprofile" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/secrules" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -3355,7 +3354,7 @@ func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.Token for _, guestdisk := range self.GetDisks() { disk := guestdisk.GetDisk() storage := disk.GetStorage() - if utils.IsInStringArray(storage.StorageType, sysutils.LOCAL_STORAGE_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) { + if utils.IsInStringArray(storage.StorageType, LOCAL_STORAGE_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) { disk.DoPendingDelete(ctx, userCred) } else { self.DetachDisk(ctx, disk, userCred) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 5371c46890..222bacc49c 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -20,7 +20,6 @@ import ( "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" @@ -483,7 +482,7 @@ func (self *SHost) GetBaremetalstorage() *SHoststorage { q := hoststorages.Query() q = q.Join(storages, sqlchemy.AND(sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")), sqlchemy.IsFalse(storages.Field("deleted")))) - q = q.Filter(sqlchemy.Equals(storages.Field("storage_type"), sysutils.STORAGE_BAREMETAL)) + q = q.Filter(sqlchemy.Equals(storages.Field("storage_type"), STORAGE_BAREMETAL)) q = q.Filter(sqlchemy.Equals(hoststorages.Field("host_id"), self.Id)) if q.Count() == 1 { hs := SHoststorage{} diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index b4261c63d7..5f63fe22ea 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -13,7 +13,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" - "yunion.io/x/pkg/util/sysutils" "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" ) @@ -807,11 +806,11 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. } if jsonutils.QueryBoolean(query, "share", false) { - q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), LOCAL_STORAGE_TYPES)) } if jsonutils.QueryBoolean(query, "local", false) { - q = q.Filter(sqlchemy.In(q.Field("storage_type"), sysutils.LOCAL_STORAGE_TYPES)) + q = q.Filter(sqlchemy.In(q.Field("storage_type"), LOCAL_STORAGE_TYPES)) } if jsonutils.QueryBoolean(query, "usable", false) { From c3026bbee1089e66856895435a6a25766b6a9912 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sun, 25 Nov 2018 23:14:40 +0800 Subject: [PATCH 04/10] update vendor --- Gopkg.lock | 27 +- .../aliyun/alibaba-cloud-sdk-go/sdk/client.go | 28 +- .../aliyun/alibaba-cloud-sdk-go/sdk/config.go | 35 ++ .../sdk/errors/client_error.go | 6 +- .../sdk/errors/server_error.go | 4 +- .../sdk/responses/json_parser.go | 341 ++++++++++++++++++ .../sdk/responses/response.go | 10 +- vendor/github.com/davecgh/go-spew/LICENSE | 2 +- .../github.com/davecgh/go-spew/spew/bypass.go | 187 +++++----- .../davecgh/go-spew/spew/bypasssafe.go | 2 +- .../github.com/davecgh/go-spew/spew/common.go | 2 +- .../github.com/davecgh/go-spew/spew/dump.go | 10 +- .../github.com/davecgh/go-spew/spew/format.go | 4 +- .../go-windows-terminal-sequences/license | 9 - vendor/yunion.io/x/jsonutils/marshal.go | 239 +++++++++--- .../x/pkg/prettytable/prettytable.go | 14 +- vendor/yunion.io/x/pkg/util/sysutils/doc.go | 1 - .../x/pkg/util/sysutils/storagetypes.go | 49 --- vendor/yunion.io/x/structarg/structarg.go | 70 +--- 19 files changed, 748 insertions(+), 292 deletions(-) create mode 100644 vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/json_parser.go delete mode 100644 vendor/github.com/konsorten/go-windows-terminal-sequences/license delete mode 100644 vendor/yunion.io/x/pkg/util/sysutils/doc.go delete mode 100644 vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go diff --git a/Gopkg.lock b/Gopkg.lock index 32230c7ea1..d82107381f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -79,7 +79,7 @@ version = "v1.19.0" [[projects]] - digest = "1:bad9acf125c8b98ea473f45cf8eabee4c401fcd108534b65d2033af7481be25d" + digest = "1:c9956ee9d99542ed5be2a37e8a35505a4ca1516520380623e3f54f51bfc53d7a" name = "github.com/aliyun/alibaba-cloud-sdk-go" packages = [ "sdk", @@ -93,8 +93,8 @@ "sdk/utils", ] pruneopts = "UT" - revision = "1d757c22fbf7997c68c43d72e92d462ea05e1439" - version = "0.7.4" + revision = "a46ca5dbc29d3df78e8da89737b5f40dd8c03d26" + version = "0.7.7" [[projects]] digest = "1:5a2d4da3f703696f4bec1ba1d363f50461b72934b1817c07ab0963496ed1c90d" @@ -248,12 +248,12 @@ version = "v1.2.0" [[projects]] - digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" + digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" name = "github.com/davecgh/go-spew" packages = ["spew"] pruneopts = "UT" - revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" - version = "v1.1.1" + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" [[projects]] digest = "1:76dc72490af7174349349838f2fe118996381b31ea83243812a97e5a0fd5ed55" @@ -1312,11 +1312,11 @@ [[projects]] branch = "master" - digest = "1:54554b3c72f4fcbd3c27c5137f9acd9d153d7e7150d77cd509af50c75fcb41b9" + digest = "1:b132a75f45cf6214d156cd2df68106fff39ca3ba4d31d27d97924fb5b5948d0c" name = "yunion.io/x/jsonutils" packages = ["."] pruneopts = "UT" - revision = "7079aada4c7e9a37e4c3b183bddb0ef684e038f2" + revision = "65d663564e6e37b6aec8864b384238a6091bdd75" [[projects]] branch = "master" @@ -1331,7 +1331,7 @@ [[projects]] branch = "master" - digest = "1:bf987f904fd821c6395fefdab956d070596eb401657fff40420014a3a8e3f45c" + digest = "1:8c4d781aafefb667511de3e0e9fe02170519f63d75a69e34045bd417f25d5523" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1355,7 +1355,6 @@ "util/secrules", "util/sets", "util/stringutils", - "util/sysutils", "util/timeutils", "util/trace", "util/ttlpool", @@ -1365,7 +1364,7 @@ "utils", ] pruneopts = "UT" - revision = "7614d751299a6703a05f757a13e6dc900332e31b" + revision = "883ea57a4b57d551bbf2fb7d1876d6ab9355c486" [[projects]] branch = "master" @@ -1377,11 +1376,11 @@ [[projects]] branch = "master" - digest = "1:f07a1ef9758f56186dd9039a8608bc9d537070a7c04479dbab6aeb42501001b7" + digest = "1:bbaf572e68e5dad4045e1e424af29838882c620df391fa1631dd8dc851c87a95" name = "yunion.io/x/structarg" packages = ["."] pruneopts = "UT" - revision = "e0cc2c73375327d401b100bd8988fa340c2687ee" + revision = "c95bf78846decd8dea379e925410f2895fb20076" [solve-meta] analyzer-name = "dep" @@ -1462,6 +1461,7 @@ "github.com/vmware/govmomi/property", "github.com/vmware/govmomi/session", "github.com/vmware/govmomi/view", + "github.com/vmware/govmomi/vim25", "github.com/vmware/govmomi/vim25/mo", "github.com/vmware/govmomi/vim25/types", "golang.org/x/crypto/ssh", @@ -1496,7 +1496,6 @@ "yunion.io/x/pkg/util/secrules", "yunion.io/x/pkg/util/sets", "yunion.io/x/pkg/util/stringutils", - "yunion.io/x/pkg/util/sysutils", "yunion.io/x/pkg/util/timeutils", "yunion.io/x/pkg/util/trace", "yunion.io/x/pkg/util/ttlpool", diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/client.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/client.go index 12548c409f..7e9a0e4204 100644 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/client.go +++ b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/client.go @@ -22,6 +22,7 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses" "net/http" + "net" ) // this value will be replaced while build: -ldflags="-X sdk.version=x.x.x" @@ -185,14 +186,35 @@ func (client *Client) DoActionWithSigner(request requests.AcsRequest, response r for key, value := range request.GetHeaders() { httpRequest.Header[key] = []string{value} } - httpResponse, err := client.httpClient.Do(httpRequest) - if err != nil { - return + var httpResponse *http.Response + for retryTimes := 0; retryTimes < client.config.MaxRetryTime; retryTimes++ { + httpResponse, err = client.httpClient.Do(httpRequest) + // if status code >= 500 or timeout, will trigger retry + if client.config.AutoRetry && isNeedRetry(httpResponse, err){ + continue + } + // receive error but not timeout + if err != nil { + return + } + break } err = responses.Unmarshal(response, httpResponse, request.GetAcceptFormat()) return } +func isNeedRetry(response *http.Response, err error) bool { + if response.StatusCode >= http.StatusInternalServerError { + // internal server error + return true + }else if err, ok := err.(net.Error); ok && err.Timeout() { + // timeout + return true + }else{ + return false + } +} + func (client *Client) AddAsyncTask(task func()) (err error) { if client.asyncTaskQueue != nil { client.asyncTaskQueue <- task diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/config.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/config.go index 52a2384987..9e9727ec57 100644 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/config.go +++ b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/config.go @@ -43,6 +43,41 @@ func (c *Config) WithTimeout(timeout time.Duration) *Config { return c } +func (c *Config) WithAutoRetry(isAutoRetry bool) *Config { + c.AutoRetry = isAutoRetry + return c +} + +func (c *Config) WithMaxRetryTime(maxRetryTime int) *Config { + c.MaxRetryTime = maxRetryTime + return c +} + +func (c *Config) WithUserAgent(userAgent string) *Config { + c.UserAgent = userAgent + return c +} + +func (c *Config) WithHttpTransport(httpTransport *http.Transport) *Config { + c.HttpTransport = httpTransport + return c +} + +func (c *Config) WithEnableAsync(isEnableAsync bool) *Config { + c.EnableAsync = isEnableAsync + return c +} + +func (c *Config) WithMaxTaskQueueSize(maxTaskQueueSize int) *Config { + c.MaxTaskQueueSize = maxTaskQueueSize + return c +} + +func (c *Config) WithGoRoutinePoolSize(goRoutinePoolSize int) *Config { + c.GoRoutinePoolSize = goRoutinePoolSize + return c +} + func (c *Config) WithDebug(isDebug bool) *Config { c.Debug = isDebug return c diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/client_error.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/client_error.go index e48f7dd0f3..d4c680f253 100644 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/client_error.go +++ b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/client_error.go @@ -14,6 +14,8 @@ package errors +import "fmt" + const ( DefaultClientErrorStatus = 400 DefaultClientErrorCode = "SDK.ClientError" @@ -54,9 +56,9 @@ func NewClientError(errorCode, message string, originErr error) Error { func (err *ClientError) Error() string { if err.originError != nil { return err.originError.Error() - } else { - return "" } + + return fmt.Sprintf("[%s] %s", err.errorCode, err.message) } func (err *ClientError) OriginError() error { diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/server_error.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/server_error.go index 9addf8a2e8..f1212ccd07 100644 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/server_error.go +++ b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors/server_error.go @@ -14,6 +14,8 @@ package errors +import "fmt" + type ServerError struct { httpStatus int errorCode string @@ -41,7 +43,7 @@ func (err *ServerError) Message() string { } func (err *ServerError) Error() string { - return "SDK.ServerError" + return fmt.Sprintf("SDK.ServerError %s %s", err.errorCode, err.message) } func (err *ServerError) OriginError() error { diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/json_parser.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/json_parser.go new file mode 100644 index 0000000000..baa7da037f --- /dev/null +++ b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/json_parser.go @@ -0,0 +1,341 @@ +package responses + +import ( + "github.com/json-iterator/go" + "sync" + "unsafe" + "encoding/json" + "strconv" + "strings" + "io" + "math" +) + +const maxUint = ^uint(0) +const maxInt = int(maxUint >> 1) +const minInt = -maxInt - 1 + +var jsonParser jsoniter.API +var initJson = &sync.Once{} + +func initJsonParserOnce() { + initJson.Do(func() { + registerBetterFuzzyDecoder() + jsonParser = jsoniter.ConfigCompatibleWithStandardLibrary + }) +} + +func registerBetterFuzzyDecoder() { + jsoniter.RegisterTypeDecoder("string", &nullableFuzzyStringDecoder{}) + jsoniter.RegisterTypeDecoder("bool", &fuzzyBoolDecoder{}) + jsoniter.RegisterTypeDecoder("float32", &nullableFuzzyFloat32Decoder{}) + jsoniter.RegisterTypeDecoder("float64", &nullableFuzzyFloat64Decoder{}) + jsoniter.RegisterTypeDecoder("int", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(maxInt) || val < float64(minInt) { + iter.ReportError("fuzzy decode int", "exceed range") + return + } + *((*int)(ptr)) = int(val) + } else { + *((*int)(ptr)) = iter.ReadInt() + } + }}) + jsoniter.RegisterTypeDecoder("uint", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(maxUint) || val < 0 { + iter.ReportError("fuzzy decode uint", "exceed range") + return + } + *((*uint)(ptr)) = uint(val) + } else { + *((*uint)(ptr)) = iter.ReadUint() + } + }}) + jsoniter.RegisterTypeDecoder("int8", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxInt8) || val < float64(math.MinInt8) { + iter.ReportError("fuzzy decode int8", "exceed range") + return + } + *((*int8)(ptr)) = int8(val) + } else { + *((*int8)(ptr)) = iter.ReadInt8() + } + }}) + jsoniter.RegisterTypeDecoder("uint8", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxUint8) || val < 0 { + iter.ReportError("fuzzy decode uint8", "exceed range") + return + } + *((*uint8)(ptr)) = uint8(val) + } else { + *((*uint8)(ptr)) = iter.ReadUint8() + } + }}) + jsoniter.RegisterTypeDecoder("int16", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxInt16) || val < float64(math.MinInt16) { + iter.ReportError("fuzzy decode int16", "exceed range") + return + } + *((*int16)(ptr)) = int16(val) + } else { + *((*int16)(ptr)) = iter.ReadInt16() + } + }}) + jsoniter.RegisterTypeDecoder("uint16", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxUint16) || val < 0 { + iter.ReportError("fuzzy decode uint16", "exceed range") + return + } + *((*uint16)(ptr)) = uint16(val) + } else { + *((*uint16)(ptr)) = iter.ReadUint16() + } + }}) + jsoniter.RegisterTypeDecoder("int32", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxInt32) || val < float64(math.MinInt32) { + iter.ReportError("fuzzy decode int32", "exceed range") + return + } + *((*int32)(ptr)) = int32(val) + } else { + *((*int32)(ptr)) = iter.ReadInt32() + } + }}) + jsoniter.RegisterTypeDecoder("uint32", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxUint32) || val < 0 { + iter.ReportError("fuzzy decode uint32", "exceed range") + return + } + *((*uint32)(ptr)) = uint32(val) + } else { + *((*uint32)(ptr)) = iter.ReadUint32() + } + }}) + jsoniter.RegisterTypeDecoder("int64", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxInt64) || val < float64(math.MinInt64) { + iter.ReportError("fuzzy decode int64", "exceed range") + return + } + *((*int64)(ptr)) = int64(val) + } else { + *((*int64)(ptr)) = iter.ReadInt64() + } + }}) + jsoniter.RegisterTypeDecoder("uint64", &nullableFuzzyIntegerDecoder{func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if isFloat { + val := iter.ReadFloat64() + if val > float64(math.MaxUint64) || val < 0 { + iter.ReportError("fuzzy decode uint64", "exceed range") + return + } + *((*uint64)(ptr)) = uint64(val) + } else { + *((*uint64)(ptr)) = iter.ReadUint64() + } + }}) +} + +type nullableFuzzyStringDecoder struct { +} + +func (decoder *nullableFuzzyStringDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + valueType := iter.WhatIsNext() + switch valueType { + case jsoniter.NumberValue: + var number json.Number + iter.ReadVal(&number) + *((*string)(ptr)) = string(number) + case jsoniter.StringValue: + *((*string)(ptr)) = iter.ReadString() + case jsoniter.BoolValue: + *((*string)(ptr)) = strconv.FormatBool(iter.ReadBool()) + case jsoniter.NilValue: + iter.ReadNil() + *((*string)(ptr)) = "" + default: + iter.ReportError("fuzzyStringDecoder", "not number or string or bool") + } +} + +type fuzzyBoolDecoder struct { +} + +func (decoder *fuzzyBoolDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + valueType := iter.WhatIsNext() + switch valueType { + case jsoniter.BoolValue: + *((*bool)(ptr)) = iter.ReadBool() + case jsoniter.NumberValue: + var number json.Number + iter.ReadVal(&number) + num, err := number.Int64() + if err != nil { + iter.ReportError("fuzzyBoolDecoder", "get value from json.number failed") + } + if num == 0 { + *((*bool)(ptr)) = false + } else { + *((*bool)(ptr)) = true + } + case jsoniter.StringValue: + strValue := strings.ToLower(iter.ReadString()) + if strValue == "true" { + *((*bool)(ptr)) = true + } else if strValue == "false" || strValue == "" { + *((*bool)(ptr)) = false + } else { + iter.ReportError("fuzzyBoolDecoder", "unsupported bool value: "+strValue) + } + case jsoniter.NilValue: + iter.ReadNil() + *((*bool)(ptr)) = false + default: + iter.ReportError("fuzzyBoolDecoder", "not number or string or nil") + } +} + +type tolerateEmptyArrayDecoder struct { + valDecoder jsoniter.ValDecoder +} + +func (decoder *tolerateEmptyArrayDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + if iter.WhatIsNext() == jsoniter.ArrayValue { + iter.Skip() + newIter := iter.Pool().BorrowIterator([]byte("{}")) + defer iter.Pool().ReturnIterator(newIter) + decoder.valDecoder.Decode(ptr, newIter) + } else { + decoder.valDecoder.Decode(ptr, iter) + } +} + +type nullableFuzzyIntegerDecoder struct { + fun func(isFloat bool, ptr unsafe.Pointer, iter *jsoniter.Iterator) +} + +func (decoder *nullableFuzzyIntegerDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + valueType := iter.WhatIsNext() + var str string + switch valueType { + case jsoniter.NumberValue: + var number json.Number + iter.ReadVal(&number) + str = string(number) + case jsoniter.StringValue: + str = iter.ReadString() + // support empty string + if str == ""{ + str = "0" + } + case jsoniter.BoolValue: + if iter.ReadBool() { + str = "1" + } else { + str = "0" + } + case jsoniter.NilValue: + iter.ReadNil() + str = "0" + default: + iter.ReportError("fuzzyIntegerDecoder", "not number or string") + } + newIter := iter.Pool().BorrowIterator([]byte(str)) + defer iter.Pool().ReturnIterator(newIter) + isFloat := strings.IndexByte(str, '.') != -1 + decoder.fun(isFloat, ptr, newIter) + if newIter.Error != nil && newIter.Error != io.EOF { + iter.Error = newIter.Error + } +} + +type nullableFuzzyFloat32Decoder struct { +} + +func (decoder *nullableFuzzyFloat32Decoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + valueType := iter.WhatIsNext() + var str string + switch valueType { + case jsoniter.NumberValue: + *((*float32)(ptr)) = iter.ReadFloat32() + case jsoniter.StringValue: + str = iter.ReadString() + // support empty string + if str == ""{ + *((*float32)(ptr)) = 0 + return + } + newIter := iter.Pool().BorrowIterator([]byte(str)) + defer iter.Pool().ReturnIterator(newIter) + *((*float32)(ptr)) = newIter.ReadFloat32() + if newIter.Error != nil && newIter.Error != io.EOF { + iter.Error = newIter.Error + } + case jsoniter.BoolValue: + // support bool to float32 + if iter.ReadBool() { + *((*float32)(ptr)) = 1 + } else { + *((*float32)(ptr)) = 0 + } + case jsoniter.NilValue: + iter.ReadNil() + *((*float32)(ptr)) = 0 + default: + iter.ReportError("nullableFuzzyFloat32Decoder", "not number or string") + } +} + +type nullableFuzzyFloat64Decoder struct { +} + +func (decoder *nullableFuzzyFloat64Decoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + valueType := iter.WhatIsNext() + var str string + switch valueType { + case jsoniter.NumberValue: + *((*float64)(ptr)) = iter.ReadFloat64() + case jsoniter.StringValue: + str = iter.ReadString() + // support empty string + if str == ""{ + *((*float64)(ptr)) = 0 + return + } + newIter := iter.Pool().BorrowIterator([]byte(str)) + defer iter.Pool().ReturnIterator(newIter) + *((*float64)(ptr)) = newIter.ReadFloat64() + if newIter.Error != nil && newIter.Error != io.EOF { + iter.Error = newIter.Error + } + case jsoniter.BoolValue: + // support bool to float64 + if iter.ReadBool() { + *((*float64)(ptr)) = 1 + } else { + *((*float64)(ptr)) = 0 + } + case jsoniter.NilValue: + // support empty string + iter.ReadNil() + *((*float64)(ptr)) = 0 + default: + iter.ReportError("nullableFuzzyFloat32Decoder", "not number or string") + } +} diff --git a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/response.go b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/response.go index 108efaaff0..2ad8412e03 100644 --- a/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/response.go +++ b/vendor/github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses/response.go @@ -16,8 +16,8 @@ package responses import ( "bytes" - "encoding/json" "encoding/xml" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors" "io/ioutil" "net/http" "strconv" @@ -25,6 +25,7 @@ import ( ) type AcsResponse interface { + IsSuccess() bool GetHttpStatus() int GetHttpHeaders() map[string][]string GetHttpContentString() string @@ -38,8 +39,13 @@ func Unmarshal(response AcsResponse, httpResponse *http.Response, format string) if err != nil { return } + if !response.IsSuccess() { + err = errors.NewServerError(response.GetHttpStatus(), response.GetOriginHttpResponse().Status, response.GetHttpContentString()) + return + } if strings.ToUpper(format) == "JSON" { - err = json.Unmarshal(response.GetHttpContentBytes(), response) + initJsonParserOnce() + err = jsonParser.Unmarshal(response.GetHttpContentBytes(), response) } else if strings.ToUpper(format) == "XML" { err = xml.Unmarshal(response.GetHttpContentBytes(), response) } diff --git a/vendor/github.com/davecgh/go-spew/LICENSE b/vendor/github.com/davecgh/go-spew/LICENSE index bc52e96f2b..c836416192 100644 --- a/vendor/github.com/davecgh/go-spew/LICENSE +++ b/vendor/github.com/davecgh/go-spew/LICENSE @@ -2,7 +2,7 @@ ISC License Copyright (c) 2012-2016 Dave Collins -Permission to use, copy, modify, and/or distribute this software for any +Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. diff --git a/vendor/github.com/davecgh/go-spew/spew/bypass.go b/vendor/github.com/davecgh/go-spew/spew/bypass.go index 792994785e..8a4a6589a2 100644 --- a/vendor/github.com/davecgh/go-spew/spew/bypass.go +++ b/vendor/github.com/davecgh/go-spew/spew/bypass.go @@ -16,9 +16,7 @@ // when the code is not running on Google App Engine, compiled by GopherJS, and // "-tags safe" is not added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. -// Go versions prior to 1.4 are disabled because they use a different layout -// for interfaces which make the implementation of unsafeReflectValue more complex. -// +build !js,!appengine,!safe,!disableunsafe,go1.4 +// +build !js,!appengine,!safe,!disableunsafe package spew @@ -36,49 +34,80 @@ const ( ptrSize = unsafe.Sizeof((*byte)(nil)) ) -type flag uintptr - var ( - // flagRO indicates whether the value field of a reflect.Value - // is read-only. - flagRO flag + // offsetPtr, offsetScalar, and offsetFlag are the offsets for the + // internal reflect.Value fields. These values are valid before golang + // commit ecccf07e7f9d which changed the format. The are also valid + // after commit 82f48826c6c7 which changed the format again to mirror + // the original format. Code in the init function updates these offsets + // as necessary. + offsetPtr = uintptr(ptrSize) + offsetScalar = uintptr(0) + offsetFlag = uintptr(ptrSize * 2) - // flagAddr indicates whether the address of the reflect.Value's - // value may be taken. - flagAddr flag + // flagKindWidth and flagKindShift indicate various bits that the + // reflect package uses internally to track kind information. + // + // flagRO indicates whether or not the value field of a reflect.Value is + // read-only. + // + // flagIndir indicates whether the value field of a reflect.Value is + // the actual data or a pointer to the data. + // + // These values are valid before golang commit 90a7c3c86944 which + // changed their positions. Code in the init function updates these + // flags as necessary. + flagKindWidth = uintptr(5) + flagKindShift = uintptr(flagKindWidth - 1) + flagRO = uintptr(1 << 0) + flagIndir = uintptr(1 << 1) ) -// flagKindMask holds the bits that make up the kind -// part of the flags field. In all the supported versions, -// it is in the lower 5 bits. -const flagKindMask = flag(0x1f) - -// Different versions of Go have used different -// bit layouts for the flags type. This table -// records the known combinations. -var okFlags = []struct { - ro, addr flag -}{{ - // From Go 1.4 to 1.5 - ro: 1 << 5, - addr: 1 << 7, -}, { - // Up to Go tip. - ro: 1<<5 | 1<<6, - addr: 1 << 8, -}} - -var flagValOffset = func() uintptr { - field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") - if !ok { - panic("reflect.Value has no flag field") +func init() { + // Older versions of reflect.Value stored small integers directly in the + // ptr field (which is named val in the older versions). Versions + // between commits ecccf07e7f9d and 82f48826c6c7 added a new field named + // scalar for this purpose which unfortunately came before the flag + // field, so the offset of the flag field is different for those + // versions. + // + // This code constructs a new reflect.Value from a known small integer + // and checks if the size of the reflect.Value struct indicates it has + // the scalar field. When it does, the offsets are updated accordingly. + vv := reflect.ValueOf(0xf00) + if unsafe.Sizeof(vv) == (ptrSize * 4) { + offsetScalar = ptrSize * 2 + offsetFlag = ptrSize * 3 } - return field.Offset -}() -// flagField returns a pointer to the flag field of a reflect.Value. -func flagField(v *reflect.Value) *flag { - return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset)) + // Commit 90a7c3c86944 changed the flag positions such that the low + // order bits are the kind. This code extracts the kind from the flags + // field and ensures it's the correct type. When it's not, the flag + // order has been changed to the newer format, so the flags are updated + // accordingly. + upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag) + upfv := *(*uintptr)(upf) + flagKindMask := uintptr((1<>flagKindShift != uintptr(reflect.Int) { + flagKindShift = 0 + flagRO = 1 << 5 + flagIndir = 1 << 6 + + // Commit adf9b30e5594 modified the flags to separate the + // flagRO flag into two bits which specifies whether or not the + // field is embedded. This causes flagIndir to move over a bit + // and means that flagRO is the combination of either of the + // original flagRO bit and the new bit. + // + // This code detects the change by extracting what used to be + // the indirect bit to ensure it's set. When it's not, the flag + // order has been changed to the newer format, so the flags are + // updated accordingly. + if upfv&flagIndir == 0 { + flagRO = 3 << 5 + flagIndir = 1 << 7 + } + } } // unsafeReflectValue converts the passed reflect.Value into a one that bypasses @@ -90,56 +119,34 @@ func flagField(v *reflect.Value) *flag { // This allows us to check for implementations of the Stringer and error // interfaces to be used for pretty printing ordinarily unaddressable and // inaccessible values such as unexported struct fields. -func unsafeReflectValue(v reflect.Value) reflect.Value { - if !v.IsValid() || (v.CanInterface() && v.CanAddr()) { - return v - } - flagFieldPtr := flagField(&v) - *flagFieldPtr &^= flagRO - *flagFieldPtr |= flagAddr - return v -} - -// Sanity checks against future reflect package changes -// to the type or semantics of the Value.flag field. -func init() { - field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") - if !ok { - panic("reflect.Value has no flag field") - } - if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() { - panic("reflect.Value flag field has changed kind") - } - type t0 int - var t struct { - A t0 - // t0 will have flagEmbedRO set. - t0 - // a will have flagStickyRO set - a t0 - } - vA := reflect.ValueOf(t).FieldByName("A") - va := reflect.ValueOf(t).FieldByName("a") - vt0 := reflect.ValueOf(t).FieldByName("t0") - - // Infer flagRO from the difference between the flags - // for the (otherwise identical) fields in t. - flagPublic := *flagField(&vA) - flagWithRO := *flagField(&va) | *flagField(&vt0) - flagRO = flagPublic ^ flagWithRO - - // Infer flagAddr from the difference between a value - // taken from a pointer and not. - vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A") - flagNoPtr := *flagField(&vA) - flagPtr := *flagField(&vPtrA) - flagAddr = flagNoPtr ^ flagPtr - - // Check that the inferred flags tally with one of the known versions. - for _, f := range okFlags { - if flagRO == f.ro && flagAddr == f.addr { - return +func unsafeReflectValue(v reflect.Value) (rv reflect.Value) { + indirects := 1 + vt := v.Type() + upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr) + rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag)) + if rvf&flagIndir != 0 { + vt = reflect.PtrTo(v.Type()) + indirects++ + } else if offsetScalar != 0 { + // The value is in the scalar field when it's not one of the + // reference types. + switch vt.Kind() { + case reflect.Uintptr: + case reflect.Chan: + case reflect.Func: + case reflect.Map: + case reflect.Ptr: + case reflect.UnsafePointer: + default: + upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + + offsetScalar) } } - panic("reflect.Value read-only flag has changed semantics") + + pv := reflect.NewAt(vt, upv) + rv = pv + for i := 0; i < indirects; i++ { + rv = rv.Elem() + } + return rv } diff --git a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go index 205c28d68c..1fe3cf3d5d 100644 --- a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go +++ b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go @@ -16,7 +16,7 @@ // when the code is running on Google App Engine, compiled by GopherJS, or // "-tags safe" is added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. -// +build js appengine safe disableunsafe !go1.4 +// +build js appengine safe disableunsafe package spew diff --git a/vendor/github.com/davecgh/go-spew/spew/common.go b/vendor/github.com/davecgh/go-spew/spew/common.go index 1be8ce9457..7c519ff47a 100644 --- a/vendor/github.com/davecgh/go-spew/spew/common.go +++ b/vendor/github.com/davecgh/go-spew/spew/common.go @@ -180,7 +180,7 @@ func printComplex(w io.Writer, c complex128, floatPrecision int) { w.Write(closeParenBytes) } -// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x' +// printHexPtr outputs a uintptr formatted as hexidecimal with a leading '0x' // prefix to Writer w. func printHexPtr(w io.Writer, p uintptr) { // Null pointer. diff --git a/vendor/github.com/davecgh/go-spew/spew/dump.go b/vendor/github.com/davecgh/go-spew/spew/dump.go index f78d89fc1f..df1d582a72 100644 --- a/vendor/github.com/davecgh/go-spew/spew/dump.go +++ b/vendor/github.com/davecgh/go-spew/spew/dump.go @@ -35,16 +35,16 @@ var ( // cCharRE is a regular expression that matches a cgo char. // It is used to detect character arrays to hexdump them. - cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`) + cCharRE = regexp.MustCompile("^.*\\._Ctype_char$") // cUnsignedCharRE is a regular expression that matches a cgo unsigned // char. It is used to detect unsigned character arrays to hexdump // them. - cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`) + cUnsignedCharRE = regexp.MustCompile("^.*\\._Ctype_unsignedchar$") // cUint8tCharRE is a regular expression that matches a cgo uint8_t. // It is used to detect uint8_t arrays to hexdump them. - cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`) + cUint8tCharRE = regexp.MustCompile("^.*\\._Ctype_uint8_t$") ) // dumpState contains information about the state of a dump operation. @@ -143,10 +143,10 @@ func (d *dumpState) dumpPtr(v reflect.Value) { // Display dereferenced value. d.w.Write(openParenBytes) switch { - case nilFound: + case nilFound == true: d.w.Write(nilAngleBytes) - case cycleFound: + case cycleFound == true: d.w.Write(circularBytes) default: diff --git a/vendor/github.com/davecgh/go-spew/spew/format.go b/vendor/github.com/davecgh/go-spew/spew/format.go index b04edb7d7a..c49875bacb 100644 --- a/vendor/github.com/davecgh/go-spew/spew/format.go +++ b/vendor/github.com/davecgh/go-spew/spew/format.go @@ -182,10 +182,10 @@ func (f *formatState) formatPtr(v reflect.Value) { // Display dereferenced value. switch { - case nilFound: + case nilFound == true: f.fs.Write(nilAngleBytes) - case cycleFound: + case cycleFound == true: f.fs.Write(circularShortBytes) default: diff --git a/vendor/github.com/konsorten/go-windows-terminal-sequences/license b/vendor/github.com/konsorten/go-windows-terminal-sequences/license deleted file mode 100644 index 14127cd831..0000000000 --- a/vendor/github.com/konsorten/go-windows-terminal-sequences/license +++ /dev/null @@ -1,9 +0,0 @@ -(The MIT License) - -Copyright (c) 2017 marvin + konsorten GmbH (open-source@konsorten.de) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/yunion.io/x/jsonutils/marshal.go b/vendor/yunion.io/x/jsonutils/marshal.go index aff79cf1d2..c18685bb2b 100644 --- a/vendor/yunion.io/x/jsonutils/marshal.go +++ b/vendor/yunion.io/x/jsonutils/marshal.go @@ -12,37 +12,106 @@ import ( "reflect" "time" + "strings" "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" - "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" ) -func marshalSlice(val reflect.Value) *JSONArray { +func marshalSlice(val reflect.Value, info *jsonMarshalInfo) JSONObject { + if val.Len() == 0 && info != nil && info.omitEmpty { + return nil + } objs := make([]JSONObject, val.Len()) for i := 0; i < val.Len(); i += 1 { - objs[i] = marshalValue(val.Index(i)) + objs[i] = marshalValue(val.Index(i), nil) + } + arr := NewArray(objs...) + if info != nil && info.forceString { + return NewString(arr.String()) + } else { + return arr } - return NewArray(objs...) } -func marshalMap(val reflect.Value) *JSONDict { +func marshalMap(val reflect.Value, info *jsonMarshalInfo) JSONObject { keys := val.MapKeys() + if len(keys) == 0 && info != nil && info.omitEmpty { + return nil + } objPairs := make([]JSONPair, 0) for i := 0; i < len(keys); i += 1 { key := keys[i] - val := marshalValue(val.MapIndex(key)) + val := marshalValue(val.MapIndex(key), nil) if val != JSONNull { objPairs = append(objPairs, JSONPair{key: fmt.Sprintf("%s", key), val: val}) } } - return NewDict(objPairs...) + dict := NewDict(objPairs...) + if info != nil && info.forceString { + return NewString(dict.String()) + } else { + return dict + } } -func marshalStruct(val reflect.Value) *JSONDict { +func marshalStruct(val reflect.Value, info *jsonMarshalInfo) JSONObject { objPairs := struct2JSONPairs(val) - return NewDict(objPairs...) + if len(objPairs) == 0 && info != nil && info.omitEmpty { + return nil + } + dict := NewDict(objPairs...) + if info != nil && info.forceString { + return NewString(dict.String()) + } else { + return dict + } +} + +type jsonMarshalInfo struct { + ignore bool + omitEmpty bool + name string + forceString bool +} + +func parseJsonMarshalInfo(fieldTag reflect.StructTag) jsonMarshalInfo { + info := jsonMarshalInfo{} + info.omitEmpty = true + + tags := utils.TagMap(fieldTag) + if val, ok := tags["json"]; ok { + keys := strings.Split(val, ",") + if len(keys) > 0 { + if keys[0] == "-" { + if len(keys) > 1 { + info.name = keys[0] + } else { + info.ignore = true + } + } else { + info.name = keys[0] + } + } + if len(keys) > 1 { + for _, k := range keys[1:] { + switch k { + case "omitempty": + info.omitEmpty = true + case "string": + info.forceString = true + case "allowempty": + info.omitEmpty = false + } + } + } + } + if val, ok := tags["name"]; ok { + info.name = val + } + return info } func struct2JSONPairs(val reflect.Value) []JSONPair { @@ -57,12 +126,17 @@ func struct2JSONPairs(val reflect.Value) []JSONPair { newPairs := struct2JSONPairs(val.Field(i)) objPairs = append(objPairs, newPairs...) } else { - key := reflectutils.GetStructFieldName(&fieldType) // utils.CamelSplit(fieldType.Name, "_") - if key == "" { + jsonInfo := parseJsonMarshalInfo(fieldType.Tag) + + if jsonInfo.ignore { continue } - val := marshalValue(val.Field(i)) - if val != JSONNull { + key := jsonInfo.name + if len(key) == 0 { + key = utils.CamelSplit(fieldType.Name, "_") + } + val := marshalValue(val.Field(i), &jsonInfo) + if val != nil && val != JSONNull { objPair := JSONPair{key: key, val: val} objPairs = append(objPairs, objPair) } @@ -71,42 +145,67 @@ func struct2JSONPairs(val reflect.Value) []JSONPair { return objPairs } -func marshalInt64(val int64) *JSONInt { - return NewInt(val) -} - -func marshalFloat64(val float64) *JSONFloat { - return NewFloat(val) -} - -func marshalBoolean(val bool) *JSONBool { - if val { - return JSONTrue +func marshalInt64(val int64, info *jsonMarshalInfo) JSONObject { + if val == 0 && info != nil && info.omitEmpty { + return nil + } else if info != nil && info.forceString { + return NewString(fmt.Sprintf("%d", val)) } else { - return JSONFalse + return NewInt(val) } } -func marshalTristate(val tristate.TriState) JSONObject { +func marshalFloat64(val float64, info *jsonMarshalInfo) JSONObject { + if val == 0.0 && info != nil && info.omitEmpty { + return nil + } else if info != nil && info.forceString { + return NewString(fmt.Sprintf("%f", val)) + } else { + return NewFloat(val) + } +} + +func marshalBoolean(val bool, info *jsonMarshalInfo) JSONObject { + if !val && info != nil && info.omitEmpty { + return nil + } else if info != nil && info.forceString { + return NewString(fmt.Sprintf("%v", val)) + } else { + if val { + return JSONTrue + } else { + return JSONFalse + } + } +} + +func marshalTristate(val tristate.TriState, info *jsonMarshalInfo) JSONObject { if val.IsTrue() { return JSONTrue } else if val.IsFalse() { return JSONFalse } else { - return JSONNull + if info != nil && info.omitEmpty { + return nil + } else { + return JSONNull + } } } -func marshalString(val string) JSONObject { - if len(val) == 0 { - return JSONNull +func marshalString(val string, info *jsonMarshalInfo) JSONObject { + if len(val) == 0 && info != nil && info.omitEmpty { + return nil } else { return NewString(val) } } -func marshalTime(val time.Time) *JSONString { +func marshalTime(val time.Time, info *jsonMarshalInfo) *JSONString { if val.IsZero() { + if info != nil && info.omitEmpty { + return nil + } return NewString("") } else { return NewString(timeutils.FullIsoTime(val)) @@ -118,94 +217,118 @@ func Marshal(obj interface{}) JSONObject { return JSONNull } objValue := reflect.Indirect(reflect.ValueOf(obj)) - return marshalValue(objValue) + return marshalValue(objValue, nil) } -func marshalValue(objValue reflect.Value) JSONObject { +func marshalValue(objValue reflect.Value, info *jsonMarshalInfo) JSONObject { switch objValue.Type() { case JSONDictPtrType, JSONArrayPtrType, JSONBoolPtrType, JSONIntPtrType, JSONFloatPtrType, JSONStringPtrType, JSONObjectType: if objValue.IsNil() { - return JSONNull + return nil } return objValue.Interface().(JSONObject) case JSONDictType: json, ok := objValue.Interface().(JSONDict) if ok { - return &json + if len(json.data) == 0 && info != nil && info.omitEmpty { + return nil + } else { + return &json + } } else { - return JSONNull + return nil } case JSONArrayType: json, ok := objValue.Interface().(JSONArray) if ok { - return &json + if len(json.data) == 0 && info != nil && info.omitEmpty { + return nil + } else { + return &json + } } else { - return JSONNull + return nil } case JSONBoolType: json, ok := objValue.Interface().(JSONBool) if ok { - return &json + if !json.data && info != nil && info.omitEmpty { + return nil + } else { + return &json + } } else { - return JSONNull + return nil } case JSONIntType: json, ok := objValue.Interface().(JSONInt) if ok { - return &json + if json.data == 0 && info != nil && info.omitEmpty { + return nil + } else { + return &json + } } else { - return JSONNull + return nil } case JSONFloatType: json, ok := objValue.Interface().(JSONFloat) if ok { - return &json + if json.data == 0.0 && info != nil && info.omitEmpty { + return nil + } else { + return &json + } } else { - return JSONNull + return nil } case JSONStringType: json, ok := objValue.Interface().(JSONString) if ok { - return &json + if len(json.data) == 0 && info != nil && info.omitEmpty { + return nil + } else { + return &json + } } else { - return JSONNull + return nil } case tristate.TriStateType: tri, ok := objValue.Interface().(tristate.TriState) if ok { - return marshalTristate(tri) + return marshalTristate(tri, info) } else { - return JSONNull + return nil } } switch objValue.Kind() { case reflect.Slice, reflect.Array: - return marshalSlice(objValue) + return marshalSlice(objValue, info) case reflect.Struct: if objValue.Type() == gotypes.TimeType { - return marshalTime(objValue.Interface().(time.Time)) + return marshalTime(objValue.Interface().(time.Time), info) } else { - return marshalStruct(objValue) + return marshalStruct(objValue, info) } case reflect.Map: - return marshalMap(objValue) + return marshalMap(objValue, info) case reflect.String: strValue := objValue.Convert(gotypes.StringType) - return marshalString(strValue.Interface().(string)) + return marshalString(strValue.Interface().(string), info) case reflect.Bool: - return marshalBoolean(objValue.Interface().(bool)) + return marshalBoolean(objValue.Interface().(bool), info) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: intValue := objValue.Convert(gotypes.Int64Type) - return marshalInt64(intValue.Interface().(int64)) + return marshalInt64(intValue.Interface().(int64), info) case reflect.Float32, reflect.Float64: floatValue := objValue.Convert(gotypes.Float64Type) - return marshalFloat64(floatValue.Interface().(float64)) + return marshalFloat64(floatValue.Interface().(float64), info) case reflect.Interface, reflect.Ptr: if objValue.IsNil() { - return JSONNull + return nil } - return marshalValue(objValue.Elem()) + return marshalValue(objValue.Elem(), info) default: log.Errorf("unsupport object %s %s", objValue.Type(), objValue.Interface()) return JSONNull diff --git a/vendor/yunion.io/x/pkg/prettytable/prettytable.go b/vendor/yunion.io/x/pkg/prettytable/prettytable.go index f5ee4f34f1..feba929161 100644 --- a/vendor/yunion.io/x/pkg/prettytable/prettytable.go +++ b/vendor/yunion.io/x/pkg/prettytable/prettytable.go @@ -3,6 +3,7 @@ package prettytable import ( "bytes" "strings" + "unicode" ) type AlignmentType uint8 @@ -102,6 +103,17 @@ func textLine(buf *bytes.Buffer, columns []ptColumn, widths []int) { } } +func runeDisplayWidth(r rune) int { + const puncts = "。，；：（）、？《》" + if unicode.Is(unicode.Han, r) { + return 2 + } + if strings.ContainsRune(puncts, r) { + return 2 + } + return 1 +} + // cellDisplayWidth returns display width of the cell when printed as the // nthCol. prevWidth is the total display width (as return by this same func) // of previous cells in the same line @@ -116,7 +128,7 @@ func cellDisplayWidth(cell string, nthCol int, prevWidth int) int { for _, c := range line { incr := 0 if c != '\t' { - incr = 1 + incr = runeDisplayWidth(c) } else { // terminal with have the char TabWidth aligned incr = TabWidth - (x & (TabWidth - 1)) diff --git a/vendor/yunion.io/x/pkg/util/sysutils/doc.go b/vendor/yunion.io/x/pkg/util/sysutils/doc.go deleted file mode 100644 index 8294259502..0000000000 --- a/vendor/yunion.io/x/pkg/util/sysutils/doc.go +++ /dev/null @@ -1 +0,0 @@ -package sysutils // import "yunion.io/x/pkg/util/sysutils" diff --git a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go deleted file mode 100644 index 3f9de2fe1b..0000000000 --- a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go +++ /dev/null @@ -1,49 +0,0 @@ -package sysutils - -const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" - - STORAGE_CLOUD = "cloud" - STORAGE_CLOUD_SSD = "cloud_ssd" - STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 - STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" - - //Azure hdd and ssd storagetype - STORAGE_STANDARD_GRS = "standard_grs" - STORAGE_STANDARD_LRS = "standard_lrs" - STORAGE_STANDARD_RAGRS = "standard_ragrs" - STORAGE_STANDARD_ZRS = "standard_zrs" - STORAGE_PREMIUM_LRS = "premium_lrs" - - // aws storage type - STORAGE_GP2_SSD = "gp2" // aws general purpose ssd - STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD - STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD - STORAGE_SC1_SSD = "sc1" // aws Cold HDD - STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes - - // qcloud storage type - // STORAGE_CLOUD_SSD ="cloud_ssd" - STORAGE_LOCAL_BASIC = "local_basic" - STORAGE_LOCAL_SSD = "local_ssd" - STORAGE_CLOUD_BASIC = "cloud_basic" - STORAGE_CLOUD_PERMIUM = "cloud_permium" -) - -var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, - STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, - STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY, - STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS, - STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD, - STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM, -} - -var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} - -// var SUPPORT = []string {support = [STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS] diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index e09e97cdad..ff749484c7 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -187,56 +187,6 @@ func (this *ArgumentParser) addStructArgument(tp reflect.Type, val reflect.Value return nil } -/*func findWord(str []byte, offset int) (string, int) { - var buffer bytes.Buffer - i := skipEmpty(str, offset) - if i >= len(str) { - return "", i - } - var endstr string - quote := false - if str[i] == '"' { - quote = true - endstr = "\"" - i++ - } else if str[i] == '\'' { - quote = true - endstr = "'" - i++ - } else { - endstr = " :,\t\n}]" - } - for i < len(str) { - if quote && str[i] == '\\' { - if i+1 < len(str) { - i++ - switch str[i] { - case 'n': - buffer.WriteByte('\n') - case 'r': - buffer.WriteByte('\r') - case 't': - buffer.WriteByte('\t') - default: - buffer.WriteByte(str[i]) - } - i++ - } else { - break - } - } else if strings.IndexByte(endstr, str[i]) >= 0 { // end - if quote { - i++ - } - break - } else { - buffer.WriteByte(str[i]) - i++ - } - } - return buffer.String(), i -}*/ - func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) error { tagMap := utils.TagMap(f.Tag) help := tagMap[TAG_HELP] @@ -901,6 +851,14 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error return err } +func isQuotedByChar(str string, quoteChar byte) bool { + return str[0] == quoteChar && str[len(str)-1] == quoteChar +} + +func isQuoted(str string) bool { + return isQuotedByChar(str, '"') || isQuotedByChar(str, '\'') +} + func (this *ArgumentParser) parseKeyValue(key, value string) error { arg := this.findOptionalArgument(key) if arg != nil { @@ -914,7 +872,15 @@ func (this *ArgumentParser) parseKeyValue(key, value string) error { } } } else { - return arg.SetValue(value) + if !isQuoted(value) { + value = fmt.Sprintf("\"%s\"", value) + } + values := utils.FindWords([]byte(value), 0) + if len(values) == 1 { + return arg.SetValue(values[0]) + } else { + log.Warningf("too many arguments %#v for %s", values, key) + } } } else { log.Warningf("Cannot find argument %s", key) @@ -964,7 +930,7 @@ func (this *ArgumentParser) ParseFile(filepath string) error { for scanner.Scan() { line := scanner.Text() line = strings.TrimSpace(removeComments(line)) - line = removeCharacters(line, `"'`) + // line = removeCharacters(line, `"'`) if len(line) > 0 { key, val, e := line2KeyValue(line) if e == nil { From 9e5aee6e6e3d5d9c3c7805cd2e8567ea90e7ddb9 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Sun, 25 Nov 2018 23:21:16 +0800 Subject: [PATCH 05/10] backport remove sysutils --- pkg/compute/guestdrivers/qcloud.go | 2 +- pkg/compute/models/guests.go | 2 +- pkg/compute/models/storages.go | 63 ++++++++++++++++++++++-------- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index e1dc7c1536..aaa267c0e9 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -72,7 +72,7 @@ func (self *SQcloudGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m if !utils.IsInStringArray(guest.Status, []string{models.VM_READY, models.VM_RUNNING}) { return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status) } - if utils.IsInStringArray(storage.StorageType, []string{sysutils.STORAGE_LOCAL_BASIC, sysutils.STORAGE_LOCAL_SSD}) { + if utils.IsInStringArray(storage.StorageType, []string{models.STORAGE_LOCAL_BASIC, models.STORAGE_LOCAL_SSD}) { return fmt.Errorf("Cannot resize %s disk", storage.StorageType) } return nil diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 660acf7093..2ba7e44889 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3354,7 +3354,7 @@ func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.Token for _, guestdisk := range self.GetDisks() { disk := guestdisk.GetDisk() storage := disk.GetStorage() - if utils.IsInStringArray(storage.StorageType, LOCAL_STORAGE_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) { + if utils.IsInStringArray(storage.StorageType, STORAGE_LOCAL_TYPES) || utils.IsInStringArray(disk.DiskType, []string{DISK_TYPE_SYS, DISK_TYPE_SWAP}) || (utils.IsInStringArray(self.Hypervisor, PUBLIC_CLOUD_HYPERVISORS) && disk.AutoDelete) { disk.DoPendingDelete(ctx, userCred) } else { self.DetachDisk(ctx, disk, userCred) diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 5f63fe22ea..0467f97216 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -18,23 +18,43 @@ import ( ) const ( - STORAGE_LOCAL = "local" - STORAGE_BAREMETAL = "baremetal" - STORAGE_SHEEPDOG = "sheepdog" - STORAGE_RBD = "rbd" - STORAGE_DOCKER = "docker" - STORAGE_NAS = "nas" - STORAGE_VSAN = "vsan" + STORAGE_LOCAL = "local" + STORAGE_BAREMETAL = "baremetal" + STORAGE_SHEEPDOG = "sheepdog" + STORAGE_RBD = "rbd" + STORAGE_DOCKER = "docker" + STORAGE_NAS = "nas" + STORAGE_VSAN = "vsan" + STORAGE_NFS = "nfs" + STORAGE_PUBLIC_CLOUD = "cloud" STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" STORAGE_CLOUD_SSD = "cloud_ssd" - STORAGE_EPHEMERAL_SSD = "ephemeral_ssd" - STORAGE_GP2_SSD = "gp2" // aws general purpose ssd - STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD - STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD - STORAGE_SC1_SSD = "sc1" // aws Cold HDD - STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes + STORAGE_CLOUD_ESSD = "cloud_essd" //增强型(Enhanced)SSD 云盘 + //Azure hdd and ssd storagetype + STORAGE_STANDARD_GRS = "standard_grs" + STORAGE_STANDARD_LRS = "standard_lrs" + STORAGE_STANDARD_RAGRS = "standard_ragrs" + STORAGE_STANDARD_ZRS = "standard_zrs" + STORAGE_PREMIUM_LRS = "premium_lrs" + + // aws storage type + STORAGE_GP2_SSD = "gp2" // aws general purpose ssd + STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD + STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD + STORAGE_SC1_SSD = "sc1" // aws Cold HDD + STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes + + // qcloud storage type + // STORAGE_CLOUD_SSD ="cloud_ssd" + STORAGE_LOCAL_BASIC = "local_basic" + STORAGE_LOCAL_SSD = "local_ssd" + STORAGE_CLOUD_BASIC = "cloud_basic" + STORAGE_CLOUD_PERMIUM = "cloud_permium" +) + +const ( STORAGE_ENABLED = "enabled" STORAGE_DISABLED = "disabled" STORAGE_OFFLINE = "offline" @@ -47,13 +67,22 @@ const ( var ( DISK_TYPES = []string{DISK_TYPE_ROTATE, DISK_TYPE_SSD, DISK_TYPE_HYBRID} - STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS} + STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} STORAGE_SUPPORT_TYPES = STORAGE_LOCAL_TYPES STORAGE_ALL_TYPES = []string{ STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, + STORAGE_NFS, } - STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD} + STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, + STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_NFS, + STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY, + STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS, + STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD, + STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM, + } + + STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS} ) type SStorageManager struct { @@ -806,11 +835,11 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. } if jsonutils.QueryBoolean(query, "share", false) { - q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), LOCAL_STORAGE_TYPES)) + q = q.Filter(sqlchemy.NotIn(q.Field("storage_type"), STORAGE_LOCAL_TYPES)) } if jsonutils.QueryBoolean(query, "local", false) { - q = q.Filter(sqlchemy.In(q.Field("storage_type"), LOCAL_STORAGE_TYPES)) + q = q.Filter(sqlchemy.In(q.Field("storage_type"), STORAGE_LOCAL_TYPES)) } if jsonutils.QueryBoolean(query, "usable", false) { From 3b2b4d59e013babf5035d0ebc6c6bb09e68ac765 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 26 Nov 2018 01:42:16 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A1.=20host=20re?= =?UTF-8?q?quest=20double=20token=202.=20cloud=20provider=20lost=20access?= =?UTF-8?q?=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/cloudproviders.go | 15 ++++++++++++++- pkg/mcclient/mcclient.go | 4 +++- pkg/mcclient/session.go | 3 ++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index e5b63489b1..cbf25ac56c 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -133,7 +133,19 @@ func (self *SCloudproviderManager) ValidateCreateData(ctx context.Context, userC return nil, httperrors.NewUnsupportOperationError("Directly creating cloudprovider is not supported, create cloudaccount instead") } +func (self *SCloudprovider) getAccessUrl() string { + if len(self.AccessUrl) > 0 { + return self.AccessUrl + } + account := self.GetCloudaccount() + return account.AccessUrl +} + func (self *SCloudprovider) getPassword() (string, error) { + if len(self.Secret) == 0 { + account := self.GetCloudaccount() + return account.getPassword() + } return utils.DescryptAESBase64(self.Id, self.Secret) } @@ -378,11 +390,12 @@ func (self *SCloudprovider) GetDriver() (cloudprovider.ICloudProvider, error) { return nil, fmt.Errorf("Cloud provider is not enabled") } + accessUrl := self.getAccessUrl() passwd, err := self.getPassword() if err != nil { return nil, err } - return cloudprovider.GetProvider(self.Id, self.Name, self.AccessUrl, self.Account, passwd, self.Provider) + return cloudprovider.GetProvider(self.Id, self.Name, accessUrl, self.Account, passwd, self.Provider) } func (self *SCloudprovider) savePassword(secret string) error { diff --git a/pkg/mcclient/mcclient.go b/pkg/mcclient/mcclient.go index 96b81240c7..28adaafbd4 100644 --- a/pkg/mcclient/mcclient.go +++ b/pkg/mcclient/mcclient.go @@ -76,7 +76,9 @@ func getDefaultHeader(header http.Header, token string) http.Header { if header == nil { header = http.Header{} } - header.Add("X-Auth-Token", token) + if len(header.Get(AUTH_TOKEN)) == 0 { + header.Add(AUTH_TOKEN, token) + } } return header } diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index fdf29f278d..6d9d3cc019 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -42,7 +42,8 @@ type ClientSession struct { func populateHeader(self *http.Header, update http.Header) { for k, v := range update { for _, vv := range v { - self.Add(k, vv) + // self.Add(k, vv) + self.Set(k, vv) } } } From c398301ebacc3e24e7880821b1eefd624ab49d07 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 26 Nov 2018 02:58:51 +0800 Subject: [PATCH 07/10] fix aliyun error code handling changes, use error message string to check message type --- pkg/util/aliyun/errors.go | 10 ++++++---- pkg/util/aliyun/instance.go | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/util/aliyun/errors.go b/pkg/util/aliyun/errors.go index cb899f91b7..e5dfbd20e9 100644 --- a/pkg/util/aliyun/errors.go +++ b/pkg/util/aliyun/errors.go @@ -1,13 +1,15 @@ package aliyun import ( - aliyunerrors "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors" + "fmt" + "strings" ) func isError(err error, code string) bool { - aliyunErr, ok := err.(aliyunerrors.Error) - if !ok { + errStr := fmt.Sprintf("%s", err) + if strings.Index(errStr, code) > 0 { + return true + } else { return false } - return aliyunErr.ErrorCode() == code } diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index d352d2a869..3566899954 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -702,6 +702,7 @@ func (self *SInstance) DeleteVM(ctx context.Context) error { log.Infof("The instance is initializing, try later ...") time.Sleep(10 * time.Second) } else { + log.Errorf("DeleteVM fail: %s", err) return err } } else { From 93b0a329d31efc62bfdcc5431ceed4a01234c982 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 26 Nov 2018 13:31:50 +0800 Subject: [PATCH 08/10] update vendor --- Gopkg.lock | 5 +- vendor/yunion.io/x/jsonutils/marshal.go | 76 ++++++++++++++----------- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index d82107381f..3f05cbfc65 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1312,11 +1312,11 @@ [[projects]] branch = "master" - digest = "1:b132a75f45cf6214d156cd2df68106fff39ca3ba4d31d27d97924fb5b5948d0c" + digest = "1:cdf0f619f0b3e37a792fb7f124bf2881d8a18cc13d693454b9731a8fa4f5fedc" name = "yunion.io/x/jsonutils" packages = ["."] pruneopts = "UT" - revision = "65d663564e6e37b6aec8864b384238a6091bdd75" + revision = "f5b4a20f593f1f49039455fe40cb1274de53ac3f" [[projects]] branch = "master" @@ -1397,7 +1397,6 @@ "github.com/Microsoft/azure-vhd-utils/vhdcore/footer", "github.com/Microsoft/azure-vhd-utils/vhdcore/validator", "github.com/aliyun/alibaba-cloud-sdk-go/sdk", - "github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors", "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests", "github.com/aliyun/aliyun-oss-go-sdk/oss", "github.com/aokoli/goutils", diff --git a/vendor/yunion.io/x/jsonutils/marshal.go b/vendor/yunion.io/x/jsonutils/marshal.go index c18685bb2b..fb99c077ec 100644 --- a/vendor/yunion.io/x/jsonutils/marshal.go +++ b/vendor/yunion.io/x/jsonutils/marshal.go @@ -22,7 +22,7 @@ import ( func marshalSlice(val reflect.Value, info *jsonMarshalInfo) JSONObject { if val.Len() == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } objs := make([]JSONObject, val.Len()) for i := 0; i < val.Len(); i += 1 { @@ -39,7 +39,7 @@ func marshalSlice(val reflect.Value, info *jsonMarshalInfo) JSONObject { func marshalMap(val reflect.Value, info *jsonMarshalInfo) JSONObject { keys := val.MapKeys() if len(keys) == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } objPairs := make([]JSONPair, 0) for i := 0; i < len(keys); i += 1 { @@ -60,7 +60,7 @@ func marshalMap(val reflect.Value, info *jsonMarshalInfo) JSONObject { func marshalStruct(val reflect.Value, info *jsonMarshalInfo) JSONObject { objPairs := struct2JSONPairs(val) if len(objPairs) == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } dict := NewDict(objPairs...) if info != nil && info.forceString { @@ -73,6 +73,8 @@ func marshalStruct(val reflect.Value, info *jsonMarshalInfo) JSONObject { type jsonMarshalInfo struct { ignore bool omitEmpty bool + omitFalse bool + omitZero bool name string forceString bool } @@ -80,6 +82,8 @@ type jsonMarshalInfo struct { func parseJsonMarshalInfo(fieldTag reflect.StructTag) jsonMarshalInfo { info := jsonMarshalInfo{} info.omitEmpty = true + info.omitZero = false + info.omitFalse = false tags := utils.TagMap(fieldTag) if val, ok := tags["json"]; ok { @@ -100,10 +104,18 @@ func parseJsonMarshalInfo(fieldTag reflect.StructTag) jsonMarshalInfo { switch k { case "omitempty": info.omitEmpty = true - case "string": - info.forceString = true case "allowempty": info.omitEmpty = false + case "omitzero": + info.omitZero = true + case "allowzero": + info.omitZero = false + case "omitfalse": + info.omitFalse = true + case "allowfalse": + info.omitFalse = false + case "string": + info.forceString = true } } } @@ -146,8 +158,8 @@ func struct2JSONPairs(val reflect.Value) []JSONPair { } func marshalInt64(val int64, info *jsonMarshalInfo) JSONObject { - if val == 0 && info != nil && info.omitEmpty { - return nil + if val == 0 && info != nil && info.omitZero { + return JSONNull } else if info != nil && info.forceString { return NewString(fmt.Sprintf("%d", val)) } else { @@ -156,8 +168,8 @@ func marshalInt64(val int64, info *jsonMarshalInfo) JSONObject { } func marshalFloat64(val float64, info *jsonMarshalInfo) JSONObject { - if val == 0.0 && info != nil && info.omitEmpty { - return nil + if val == 0.0 && info != nil && info.omitZero { + return JSONNull } else if info != nil && info.forceString { return NewString(fmt.Sprintf("%f", val)) } else { @@ -166,8 +178,8 @@ func marshalFloat64(val float64, info *jsonMarshalInfo) JSONObject { } func marshalBoolean(val bool, info *jsonMarshalInfo) JSONObject { - if !val && info != nil && info.omitEmpty { - return nil + if !val && info != nil && info.omitFalse { + return JSONNull } else if info != nil && info.forceString { return NewString(fmt.Sprintf("%v", val)) } else { @@ -185,26 +197,22 @@ func marshalTristate(val tristate.TriState, info *jsonMarshalInfo) JSONObject { } else if val.IsFalse() { return JSONFalse } else { - if info != nil && info.omitEmpty { - return nil - } else { - return JSONNull - } + return JSONNull } } func marshalString(val string, info *jsonMarshalInfo) JSONObject { if len(val) == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } else { return NewString(val) } } -func marshalTime(val time.Time, info *jsonMarshalInfo) *JSONString { +func marshalTime(val time.Time, info *jsonMarshalInfo) JSONObject { if val.IsZero() { if info != nil && info.omitEmpty { - return nil + return JSONNull } return NewString("") } else { @@ -224,81 +232,81 @@ func marshalValue(objValue reflect.Value, info *jsonMarshalInfo) JSONObject { switch objValue.Type() { case JSONDictPtrType, JSONArrayPtrType, JSONBoolPtrType, JSONIntPtrType, JSONFloatPtrType, JSONStringPtrType, JSONObjectType: if objValue.IsNil() { - return nil + return JSONNull } return objValue.Interface().(JSONObject) case JSONDictType: json, ok := objValue.Interface().(JSONDict) if ok { if len(json.data) == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } else { return &json } } else { - return nil + return JSONNull } case JSONArrayType: json, ok := objValue.Interface().(JSONArray) if ok { if len(json.data) == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } else { return &json } } else { - return nil + return JSONNull } case JSONBoolType: json, ok := objValue.Interface().(JSONBool) if ok { if !json.data && info != nil && info.omitEmpty { - return nil + return JSONNull } else { return &json } } else { - return nil + return JSONNull } case JSONIntType: json, ok := objValue.Interface().(JSONInt) if ok { if json.data == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } else { return &json } } else { - return nil + return JSONNull } case JSONFloatType: json, ok := objValue.Interface().(JSONFloat) if ok { if json.data == 0.0 && info != nil && info.omitEmpty { - return nil + return JSONNull } else { return &json } } else { - return nil + return JSONNull } case JSONStringType: json, ok := objValue.Interface().(JSONString) if ok { if len(json.data) == 0 && info != nil && info.omitEmpty { - return nil + return JSONNull } else { return &json } } else { - return nil + return JSONNull } case tristate.TriStateType: tri, ok := objValue.Interface().(tristate.TriState) if ok { return marshalTristate(tri, info) } else { - return nil + return JSONNull } } switch objValue.Kind() { @@ -326,7 +334,7 @@ func marshalValue(objValue reflect.Value, info *jsonMarshalInfo) JSONObject { return marshalFloat64(floatValue.Interface().(float64), info) case reflect.Interface, reflect.Ptr: if objValue.IsNil() { - return nil + return JSONNull } return marshalValue(objValue.Elem(), info) default: From 7a8210900413bcfdea14b43198e0c2ca741fb466 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 26 Nov 2018 16:24:21 +0800 Subject: [PATCH 09/10] minor fixes --- pkg/compute/models/guest_actions.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index adf3dce2bc..39af16e6d8 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -1473,11 +1473,16 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr kwargs.Set("delete_public_key", jsonutils.NewString(okey.PublicKey)) } - self.GetModelManager().TableSpec().Update(self, func() error { + _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.KeypairId = kpId return nil }) + if err != nil { + log.Errorf("update keypair fail: %s", err) + return nil, httperrors.NewInternalServerError(err.Error()) + } + kwargs.Set("reset_password", jsonutils.JSONTrue) } } From aaaba6002fb2773d7e05f2353abad494addb3c18 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 26 Nov 2018 23:21:11 +0800 Subject: [PATCH 10/10] fix aliyun fail to attach disk after create disk --- pkg/compute/guestdrivers/aliyun.go | 18 +++----- pkg/compute/models/guest_actions.go | 2 +- pkg/compute/tasks/guest_create_disk_task.go | 50 +++++++-------------- pkg/util/aliyun/disk.go | 25 +++++++---- 4 files changed, 39 insertions(+), 56 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index c973c43c75..5d86c3a783 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -98,10 +98,6 @@ func (self *SAliyunGuestDriver) ValidateCreateData(ctx context.Context, userCred func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams()) log.Debugf("RequestDeployGuestOnHost: %s", config) - /* onfinish, err := config.GetString("on_finish") - if err != nil { - return err - } */ action, err := config.GetString("action") if err != nil { @@ -263,20 +259,20 @@ func (self *SAliyunGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gu log.Errorf("fail to find VM idisks %s", err) return nil, err } - if len(idisks) < len(desc.DataDisks)+1 { + if len(idisks) < len(desc.DataDisks)+1 || idisks[0].GetGlobalId() != diskId { if waited > maxWaitSecs { log.Errorf("inconsistent disk number, wait timeout, must be something wrong on remote") return nil, cloudprovider.ErrTimeout } - log.Debugf("inconsistent disk number???? %d != %d", len(idisks), len(desc.DataDisks)+1) + if len(idisks) < len(desc.DataDisks)+1 { + log.Debugf("inconsistent disk number???? %d != %d", len(idisks), len(desc.DataDisks)+1) + } + if idisks[0].GetGlobalId() != diskId { + log.Errorf("system disk id inconsistent %s != %s", idisks[0].GetGlobalId(), diskId) + } time.Sleep(time.Second * 5) waited += 5 } else { - if idisks[0].GetGlobalId() != diskId { - log.Errorf("system disk id inconsistent %s != %s", idisks[0].GetGlobalId(), diskId) - return nil, fmt.Errorf("inconsistent sys disk id after rebuild root") - } - break } } diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 39af16e6d8..4de892036f 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -696,7 +696,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok logclient.AddActionLog(self, logclient.ACT_CREATE, "No valid storage on current host", userCred, false) return nil, httperrors.NewBadRequestError("No valid storage on current host") } - if storage.GetCapacity() < size { + if storage.GetCapacity() > 0 && storage.GetCapacity() < size { logclient.AddActionLog(self, logclient.ACT_CREATE, "Not eough storage space on current host", userCred, false) return nil, httperrors.NewBadRequestError("Not eough storage space on current host") } diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index 23fb020749..702e997a14 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "time" ) type GuestCreateDiskTask struct { @@ -130,58 +131,48 @@ func (self *ManagedGuestCreateDiskTask) OnInit(ctx context.Context, obj db.IStan func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { var diskIndex = 0 - var diskReady = true - for { + + for self.Params.Contains(fmt.Sprintf("disk.%d.id", diskIndex)) { diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if !diskReady || err != nil { - break + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return } iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) return } - if iDisk == nil { - self.SetStageFailed(ctx, "Disk not found") - return - } disk := iDisk.(*models.SDisk) if disk.Status == models.DISK_INIT { - snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) - if err != nil { - snapInfo = "" - } + snapInfo, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) return } - diskReady = false - break + return } diskIndex += 1 } + diskIndex = 0 guest := obj.(*models.SGuest) - for { + for self.Params.Contains(fmt.Sprintf("disk.%d.id", diskIndex)) { diskId, err := self.Params.GetString(fmt.Sprintf("disk.%d.id", diskIndex)) - if !diskReady || err != nil { - break + if err != nil { + return } iDisk, err := models.DiskManager.FetchById(diskId) if err != nil { self.SetStageFailed(ctx, err.Error()) return } - if iDisk == nil { - self.SetStageFailed(ctx, "Disk not found") - return - } disk := iDisk.(*models.SDisk) if disk.Status != models.DISK_READY { - diskReady = false - break + self.SetStageFailed(ctx, fmt.Sprintf("disk %s is not ready", disk.Id)) + return } iVM, e := guest.GetIVM() @@ -196,21 +187,10 @@ func (self *ManagedGuestCreateDiskTask) OnManagedDiskPrepared(ctx context.Contex self.SetStageFailed(ctx, "Attach Disk to guest fail") return } + time.Sleep(time.Second*5) diskIndex += 1 } - /*if diskReady { - if guest.Status == models.VM_RUNNING { - self.SetStage("on_config_sync_complete", nil) - err := guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) - if err != nil { - self.SetStageFailed(ctx, err.Error()) - } - } else { - - } - }*/ - self.SetStageComplete(ctx, nil) } diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index 0f1507dd45..4cf6ec9361 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -119,7 +119,22 @@ func (self *SDisk) Delete(ctx context.Context) error { log.Errorf("Failed to find disk %s when delete: %s", self.DiskId, err) return err } - return self.storage.zone.region.deleteDisk(self.DiskId) + + for { + err := self.storage.zone.region.DeleteDisk(self.DiskId) + if err != nil { + if isError(err, "IncorrectDiskStatus") { + log.Infof("The disk is initializing, try later ...") + time.Sleep(10 * time.Second) + } else { + log.Errorf("DeleteDisk fail: %s", err) + return err + } + } else { + break + } + } + return cloudprovider.WaitDeleted(self, 10*time.Second, 300*time.Second) // 5minutes } func (self *SDisk) Resize(ctx context.Context, sizeMb int64) error { @@ -246,14 +261,6 @@ func (self *SRegion) getDisk(diskId string) (*SDisk, error) { return &disks[0], nil } -func (self *SRegion) deleteDisk(diskId string) error { - params := make(map[string]string) - params["DiskId"] = diskId - - _, err := self.ecsRequest("DeleteDisk", params) - return err -} - func (self *SRegion) DeleteDisk(diskId string) error { params := make(map[string]string) params["DiskId"] = diskId