From da559be138201df0ecf2629287f3dc69179bf09c Mon Sep 17 00:00:00 2001 From: TangBin Date: Tue, 11 Dec 2018 17:52:28 +0800 Subject: [PATCH 01/34] update sku create & update & list method --- cmd/climc/shell/skus.go | 8 +- pkg/appsrv/handlerinfo.go | 2 +- pkg/compute/models/hosts.go | 2 +- pkg/compute/models/skus.go | 156 ++++++++++++++++++++++++++++++++---- 4 files changed, 145 insertions(+), 23 deletions(-) diff --git a/cmd/climc/shell/skus.go b/cmd/climc/shell/skus.go index 3fae33b88c..794f052e49 100644 --- a/cmd/climc/shell/skus.go +++ b/cmd/climc/shell/skus.go @@ -50,10 +50,9 @@ func init() { }) type ServerSkusCreateOptions struct { - Name string `help:"Name ID of SKU" required:"true" positional:"true"` CpuCoreCount int `help:"Cpu Count" required:"true" positional:"true"` MemorySizeMB int `help:"Memory MB" required:"true" positional:"true"` - Provider string `help:"Provider name" choices:"kvm|esxi"` + Provider string `help:"Provider name" choices:"all|kvm|esxi"` OsName *string `help:"OS name/type" choices:"Linux|Windows|Any" default:"Any"` SkuFamily *string `help:"sku family"` @@ -95,9 +94,8 @@ func init() { type ServerSkusUpdateOptions struct { ID string `help:"Name or ID of SKU" json:"-"` - Name *string `help:"new name of SKU"` - CpuCoreCount *int `help:"Cpu Count"` - MemorySizeMB *int `help:"Memory MB"` + CpuCoreCount *int `help:"Cpu Count"` + MemorySizeMB *int `help:"Memory MB"` SkuFamily *string `help:"sku family"` SkuCategory *string `help:"sku category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"` diff --git a/pkg/appsrv/handlerinfo.go b/pkg/appsrv/handlerinfo.go index 6a7234ca65..048a4124cd 100644 --- a/pkg/appsrv/handlerinfo.go +++ b/pkg/appsrv/handlerinfo.go @@ -110,4 +110,4 @@ func (hi *SHandlerInfo) GetAppParams(params map[string]string, path []string) *S appParams.Params = params appParams.Path = path return &appParams -} \ No newline at end of file +} diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 9fa926df40..d9107bdbfe 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -35,7 +36,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/onecloud/pkg/appsrv" ) const ( diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index 7b953b445b..7606c156da 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "fmt" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -25,6 +26,17 @@ const ( SkuCategoryHighMemory = "high_memory" // 高内存型 ) +var InstanceFamilies map[string]string = map[string]string{ + SkuCategoryGeneralPurpose: "g1", + SkuCategoryBurstable: "t1", + SkuCategoryComputeOptimized: "c1", + SkuCategoryMemoryOptimized: "r1", + SkuCategoryStorageIOOptimized: "i1", + SkuCategoryHardwareAccelerated: "", + SkuCategoryHighStorage: "hc1", + SkuCategoryHighMemory: "hr1", +} + type SServerSkuManager struct { db.SStandaloneResourceBaseManager } @@ -87,13 +99,25 @@ func inWhiteList(provider string) bool { return true } switch provider { - case HYPERVISOR_ESXI, HYPERVISOR_KVM: + case HYPERVISOR_ESXI, HYPERVISOR_KVM, "all": // 空或者all时。表示`通用`私用云instance type列表 return true default: return false } } +func genInstanceType(family string, cpu, mem_mb int64) (string, error) { + if cpu < 0 { + return "", fmt.Errorf("cpu_core_count should great than zero") + } + + if mem_mb < 0 || mem_mb%1024 != 0 { + return "", fmt.Errorf("memory_size_mb should great than zero. and should be integral multiple of 1024") + } + + return fmt.Sprintf("ecs.%s.c%dm%d", family, cpu, mem_mb/1024), nil +} + func (self *SServerSkuManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { return true } @@ -143,6 +167,50 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, } data.Add(jsonutils.NewString(zoneObj.GetId()), "zone_id") } + + // name 由服务器端生成 + _, err := data.GetString("name") + if err != nil { + data.Remove("name") + } + + cpu, err := data.Int("cpu_core_count") + if err != nil { + return nil, httperrors.NewInputParameterError("cpu_core_count should not be empty") + } + + mem, err := data.Int("memory_size_mb") + if err != nil { + return nil, httperrors.NewInputParameterError("memory_size_mb should not be empty") + } + + category, _ := data.GetString("instance_type_category") + family, exists := InstanceFamilies[category] + if !exists { + return nil, httperrors.NewInputParameterError("instance_type_category %s is invalid", category) + } + // 格式 ecs.g1.c1m1 + name, err := genInstanceType(family, cpu, mem) + if err != nil { + return nil, httperrors.NewInputParameterError(err.Error()) + } + + data.Set("name", jsonutils.NewString(name)) + + q := self.Query().Equals("name", name) + if len(provider) > 0 { + q = q.Equals("provider", provider) + } else { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + )) + } + + if q.Count() > 0 { + return nil, httperrors.NewDuplicateResourceError("Duplicate sku") + } + return self.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) } @@ -184,7 +252,15 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use q := self.Query() zone, err := query.GetString("zone") if err == nil && len(zone) > 0 { - q = q.Equals("zone_id", zone) + zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zone) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), zone) + } + return nil, httperrors.NewGeneralError(err) + } + + q = q.Equals("zone_id", zoneObj.GetId()) } else { return nil, httperrors.NewMissingParameterError("zone") } @@ -255,12 +331,12 @@ func (self *SServerSku) ValidateUpdateData( data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { if !inWhiteList(self.Provider) { - return nil, httperrors.NewForbiddenError("can not create instance_type for public cloud %s", self.Provider) + return nil, httperrors.NewForbiddenError("can not update instance_type for public cloud %s", self.Provider) } provider, err := data.GetString("provider") if err == nil && !inWhiteList(provider) { - return nil, httperrors.NewForbiddenError("can not create instance_type for public cloud %s", provider) + return nil, httperrors.NewForbiddenError("can not update instance_type for public cloud %s", provider) } zoneStr := jsonutils.GetAnyString(data, []string{"zone", "zone_id"}) @@ -274,6 +350,50 @@ func (self *SServerSku) ValidateUpdateData( } data.Add(jsonutils.NewString(zoneObj.GetId()), "zone_id") } + + // name 由服务器端生成 + _, err = data.GetString("name") + if err != nil { + data.Remove("name") + } + + cpu, err := data.Int("cpu_core_count") + if err != nil { + cpu = int64(self.CpuCoreCount) + } + + mem, err := data.Int("memory_size_mb") + if err != nil { + mem = int64(self.MemorySizeMB) + } + + category, _ := data.GetString("instance_type_category") + family, exists := InstanceFamilies[category] + if !exists { + return nil, httperrors.NewInputParameterError("instance_type_category %s is invalid", category) + } + // 格式 ecs.g1.c1m1 + name, err := genInstanceType(family, cpu, mem) + if err != nil { + return nil, httperrors.NewInputParameterError(err.Error()) + } + + data.Set("name", jsonutils.NewString(name)) + + q := self.GetModelManager().Query().Equals("name", name) + if len(provider) > 0 { + q = q.Equals("provider", provider) + } else { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + )) + } + + if q.Count() > 0 { + return nil, httperrors.NewDuplicateResourceError("sku cpu %s mem %s(Mb) already exists", cpu, mem) + } + return self.SStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data) } @@ -303,20 +423,23 @@ func (self *SServerSku) GetZoneExternalId() (string, error) { } func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { - queryDict := query.(*jsonutils.JSONDict) - provider := jsonutils.GetAnyString(query, []string{"provider"}) - if len(provider) > 0 { - if provider != "all" { - q = q.Equals("provider", provider) + if inWhiteList(provider) { + // provider 参数为空或者all时。表示查询`通用`私用云instance type列表 + if provider == "" || provider == "all" { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + )) + } else { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + sqlchemy.Equals(q.Field("provider"), provider), + )) } - - queryDict.Remove("provider") } else { - q = q.Filter(sqlchemy.OR( - sqlchemy.IsNull(q.Field("provider")), - sqlchemy.IsEmpty(q.Field("provider")), - )) + q = q.Equals("provider", provider) } q, err := manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query) @@ -336,8 +459,9 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem q = q.Equals("cloudregion_id", regionObj.GetId()) } + // 当查询私有云时，需要忽略zone参数 zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) - if len(zoneStr) > 0 { + if !inWhiteList(provider) && len(zoneStr) > 0 { zoneObj, err := ZoneManager.FetchByIdOrName(nil, zoneStr) if err != nil { if err == sql.ErrNoRows { From 9318bf6683cfd64f4cc40f84af74718d5f404b6b Mon Sep 17 00:00:00 2001 From: TangBin Date: Tue, 11 Dec 2018 19:36:21 +0800 Subject: [PATCH 02/34] update sku create & update & list method --- cmd/climc/shell/skus.go | 9 +++---- pkg/compute/models/skus.go | 50 +++++++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/cmd/climc/shell/skus.go b/cmd/climc/shell/skus.go index 794f052e49..28d7e56950 100644 --- a/cmd/climc/shell/skus.go +++ b/cmd/climc/shell/skus.go @@ -54,9 +54,8 @@ func init() { MemorySizeMB int `help:"Memory MB" required:"true" positional:"true"` Provider string `help:"Provider name" choices:"all|kvm|esxi"` - OsName *string `help:"OS name/type" choices:"Linux|Windows|Any" default:"Any"` - SkuFamily *string `help:"sku family"` - SkuCategory *string `help:"sku category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"` + OsName *string `help:"OS name/type" choices:"Linux|Windows|Any" default:"Any"` + InstanceTypeCategory *string `help:"instance type category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"` SysDiskResizable *bool `help:"system disk is resizable"` SysDiskType *string `help:"system disk type" default:"local" choices:"local"` @@ -97,8 +96,8 @@ func init() { CpuCoreCount *int `help:"Cpu Count"` MemorySizeMB *int `help:"Memory MB"` - SkuFamily *string `help:"sku family"` - SkuCategory *string `help:"sku category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"` + Provider string `help:"Provider name" choices:"all|kvm|esxi"` + InstanceTypeCategory *string `help:"instance type category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"` SysDiskResizable *bool `help:"system disk is resizable"` SysDiskMaxSizeGB *int `help:"system disk maximal size in gb"` diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index 7606c156da..e463b6bfd4 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -143,6 +143,10 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, return nil, httperrors.NewForbiddenError("can not create instance_type for public cloud %s", provider) } + if provider == "all" { + data.Remove("provider") + } + regionStr := jsonutils.GetAnyString(data, []string{"region", "region_id", "cloudregion", "cloudregion_id"}) if len(regionStr) > 0 { regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionStr) @@ -177,11 +181,15 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, cpu, err := data.Int("cpu_core_count") if err != nil { return nil, httperrors.NewInputParameterError("cpu_core_count should not be empty") + } else { + data.Set("cpu_core_count", jsonutils.NewInt(cpu)) } mem, err := data.Int("memory_size_mb") if err != nil { return nil, httperrors.NewInputParameterError("memory_size_mb should not be empty") + } else { + data.Set("cpu_core_count", jsonutils.NewInt(mem)) } category, _ := data.GetString("instance_type_category") @@ -189,6 +197,8 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, if !exists { return nil, httperrors.NewInputParameterError("instance_type_category %s is invalid", category) } + + data.Set("instance_type_family", jsonutils.NewString(family)) // 格式 ecs.g1.c1m1 name, err := genInstanceType(family, cpu, mem) if err != nil { @@ -198,7 +208,7 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, data.Set("name", jsonutils.NewString(name)) q := self.Query().Equals("name", name) - if len(provider) > 0 { + if len(provider) > 0 && provider != "all" { q = q.Equals("provider", provider) } else { q = q.Filter(sqlchemy.OR( @@ -208,7 +218,7 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, } if q.Count() > 0 { - return nil, httperrors.NewDuplicateResourceError("Duplicate sku") + return nil, httperrors.NewDuplicateResourceError("Duplicate sku %s", name) } return self.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) @@ -339,6 +349,10 @@ func (self *SServerSku) ValidateUpdateData( return nil, httperrors.NewForbiddenError("can not update instance_type for public cloud %s", provider) } + if provider == "all" { + data.Remove("provider") + } + zoneStr := jsonutils.GetAnyString(data, []string{"zone", "zone_id"}) if len(zoneStr) > 0 { zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zoneStr) @@ -361,17 +375,28 @@ func (self *SServerSku) ValidateUpdateData( if err != nil { cpu = int64(self.CpuCoreCount) } + data.Set("cpu_core_count", jsonutils.NewInt(cpu)) mem, err := data.Int("memory_size_mb") if err != nil { mem = int64(self.MemorySizeMB) } + data.Set("memory_size_mb", jsonutils.NewInt(mem)) - category, _ := data.GetString("instance_type_category") - family, exists := InstanceFamilies[category] - if !exists { - return nil, httperrors.NewInputParameterError("instance_type_category %s is invalid", category) + category, err := data.GetString("instance_type_category") + family := "" + if err != nil { + family = self.InstanceTypeFamily + } else { + f, exists := InstanceFamilies[category] + if !exists { + return nil, httperrors.NewInputParameterError("instance_type_category %s is invalid", category) + } + + family = f } + + data.Set("instance_type_family", jsonutils.NewString(family)) // 格式 ecs.g1.c1m1 name, err := genInstanceType(family, cpu, mem) if err != nil { @@ -381,7 +406,7 @@ func (self *SServerSku) ValidateUpdateData( data.Set("name", jsonutils.NewString(name)) q := self.GetModelManager().Query().Equals("name", name) - if len(provider) > 0 { + if len(provider) > 0 && provider != "all" { q = q.Equals("provider", provider) } else { q = q.Filter(sqlchemy.OR( @@ -391,7 +416,7 @@ func (self *SServerSku) ValidateUpdateData( } if q.Count() > 0 { - return nil, httperrors.NewDuplicateResourceError("sku cpu %s mem %s(Mb) already exists", cpu, mem) + return nil, httperrors.NewDuplicateResourceError("sku cpu %d mem %d(Mb) already exists", cpu, mem) } return self.SStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data) @@ -426,7 +451,14 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem provider := jsonutils.GetAnyString(query, []string{"provider"}) if inWhiteList(provider) { // provider 参数为空或者all时。表示查询`通用`私用云instance type列表 - if provider == "" || provider == "all" { + if provider == "" { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + sqlchemy.Equals(q.Field("provider"), HYPERVISOR_KVM), + sqlchemy.Equals(q.Field("provider"), HYPERVISOR_ESXI), + )) + } else if provider == "all" { q = q.Filter(sqlchemy.OR( sqlchemy.IsNull(q.Field("provider")), sqlchemy.IsEmpty(q.Field("provider")), From 6a6806e65b9bfc0048608c2a4d2555c3336a2c4b Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Thu, 6 Dec 2018 13:31:59 +0800 Subject: [PATCH 03/34] server create disk by snapshot, mount snapshot by fusefs --- Gopkg.lock | 1 + Gopkg.toml | 3 + cmd/host-image/main.go | 7 + pkg/appsrv/response.go | 11 + pkg/compute/hostdrivers/kvm.go | 16 ++ pkg/compute/models/disks.go | 101 ++++++-- pkg/compute/models/guests.go | 10 + pkg/compute/models/hosts.go | 2 +- pkg/compute/options/options.go | 2 + pkg/compute/tasks/guest_create_disk_task.go | 10 +- pkg/hostimage/host_image_service.go | 234 ++++++++++++++++++ pkg/hostimage/image.go | 134 ++++++++++ .../x/pkg/util/osprofile/osprofile.go | 2 +- vendor/yunion.io/x/sqlchemy/update.go | 14 +- 14 files changed, 510 insertions(+), 37 deletions(-) create mode 100644 cmd/host-image/main.go create mode 100644 pkg/hostimage/host_image_service.go create mode 100644 pkg/hostimage/image.go diff --git a/Gopkg.lock b/Gopkg.lock index 7c2cfbfbda..6752a09d06 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1450,6 +1450,7 @@ "github.com/miekg/dns", "github.com/moul/http2curl", "github.com/nelsonken/cos-go-sdk-v5/cos", + "github.com/pierrec/lz4", "github.com/serialx/hashring", "github.com/stretchr/testify/assert", "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common", diff --git a/Gopkg.toml b/Gopkg.toml index 64685308fd..fcf3971f99 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -24,6 +24,9 @@ # go-tests = true # unused-packages = true +[[constraint]] + name = "github.com/pierrec/lz4" + version = "2.0.7" [[constraint]] name = "github.com/360EntSecGroup-Skylar/excelize" diff --git a/cmd/host-image/main.go b/cmd/host-image/main.go new file mode 100644 index 0000000000..058ac20184 --- /dev/null +++ b/cmd/host-image/main.go @@ -0,0 +1,7 @@ +package main + +import "yunion.io/x/onecloud/pkg/hostimage" + +func main() { + hostimage.StartService() +} diff --git a/pkg/appsrv/response.go b/pkg/appsrv/response.go index f5ce722db6..ac611719af 100644 --- a/pkg/appsrv/response.go +++ b/pkg/appsrv/response.go @@ -5,6 +5,7 @@ import ( "net/http" "fmt" + "yunion.io/x/onecloud/pkg/httperrors" ) @@ -56,6 +57,16 @@ func (w *responseWriterChannel) WriteHeader(status int) { <-w.statusResp } +// implent http.Flusher +func (w *responseWriterChannel) Flush() { + if w.isClosed { + return + } + if f, ok := w.backend.(http.Flusher); ok { + f.Flush() + } +} + func (w *responseWriterChannel) wait(ctx context.Context, workerChan chan *SWorker) interface{} { var err error var worker *SWorker diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 8b6cdde8f4..83e33fc4eb 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -120,6 +120,22 @@ func (self *SKVMHostDriver) RequestUncacheImage(ctx context.Context, host *model func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { header := task.GetTaskRequestHeader() + if snapshotId, err := content.GetString("snapshot"); err == nil { + iSnapshot, _ := models.SnapshotManager.FetchById(snapshotId) + snapshot := iSnapshot.(*models.SSnapshot) + snapshotStorage := models.StorageManager.FetchStorageById(snapshot.StorageId) + snapshotHost := snapshotStorage.GetMasterHost() + if options.Options.SnapshotCreateDiskProtocol == "url" { + content.Set("snapshot_url", + jsonutils.NewString(fmt.Sprintf("%s/download/snapshots/%s/%s/%s", + snapshotHost.ManagerUri, snapshotStorage.Id, snapshot.DiskId, snapshot.Id))) + content.Set("snapshot_out_of_chain", jsonutils.NewBool(snapshot.OutOfChain)) + } else if options.Options.SnapshotCreateDiskProtocol == "fuse" { + content.Set("snapshot_url", jsonutils.NewString(fmt.Sprintf("%s/snapshots/%s/%s", + snapshotHost.GetFetchUrl(), snapshot.DiskId, snapshot.Id))) + } + content.Set("protocol", jsonutils.NewString(options.Options.SnapshotCreateDiskProtocol)) + } url := fmt.Sprintf("/disks/%s/create/%s", storage.Id, disk.Id) body := jsonutils.NewDict() diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 1f43ae83e7..f80c298bd5 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -113,6 +113,15 @@ func (manager *SDiskManager) GetContextManager() []db.IModelManager { return []db.IModelManager{StorageManager} } +func (manager *SDiskManager) FetchDiskById(diskId string) *SDisk { + disk, err := manager.FetchById(diskId) + if err != nil { + log.Errorf("FetchById fail %s", err) + return nil + } + return disk.(*SDisk) +} + func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { queryDict, ok := query.(*jsonutils.JSONDict) if !ok { @@ -1006,6 +1015,10 @@ func totalDiskSize(projectId string, active tristate.TriState, ready tristate.Tr type SDiskConfig struct { ImageId string + + SnapshotId string + DiskType string // sys, data, swap + // ImageDiskFormat string SizeMb int // MB Fs string // file system @@ -1062,28 +1075,16 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info diskConfig.SizeMb = -1 } else if utils.IsInStringArray(p, STORAGE_TYPES) { diskConfig.Backend = p + } else if strings.HasPrefix(p, "snapshot-") { + // HACK: use snapshot creat disk format snapshot-id + // example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b + log.Errorln("The snapshot XXXXXXX Create disk", p[len("snapshot-"):]) + if err := fillDiskConfigBySnapshot(userCred, &diskConfig, p[len("snapshot-"):]); err != nil { + return nil, err + } } else if len(p) > 0 { - if userCred == nil { - diskConfig.ImageId = p - } else { - image, err := CachedimageManager.getImageInfo(ctx, userCred, p, false) - if err != nil { - log.Errorf("getImageInfo fail %s", err) - return nil, err - } - if image.Status != IMAGE_STATUS_ACTIVE { - return nil, httperrors.NewInvalidStatusError("Image status is not active") - } - diskConfig.ImageId = image.Id - diskConfig.ImageProperties = image.Properties - if len(diskConfig.Format) == 0 { - diskConfig.Format = image.DiskFormat - } - // diskConfig.ImageDiskFormat = image.DiskFormat - CachedimageManager.ImageAddRefCount(image.Id) - if diskConfig.SizeMb == 0 { - diskConfig.SizeMb = image.MinDisk // MB - } + if err := fillDiskConfigByImage(ctx, userCred, &diskConfig, p); err != nil { + return nil, err } } } @@ -1095,6 +1096,60 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info return &diskConfig, nil } +func fillDiskConfigBySnapshot(userCred mcclient.TokenCredential, diskConfig *SDiskConfig, snapshotId string) error { + iSnapshot, err := SnapshotManager.FetchByIdOrName(userCred, snapshotId) + if err != nil { + if err == sql.ErrNoRows { + return httperrors.NewNotFoundError("Snapshot %s not found", snapshotId) + } + return err + } + var snapshot = iSnapshot.(*SSnapshot) + if storage := StorageManager.FetchStorageById(snapshot.StorageId); storage == nil { + return httperrors.NewBadRequestError("Snapshot %s storage %s not found, is public cloud?", + snapshotId, snapshot.StorageId) + } else { + if disk := DiskManager.FetchDiskById(snapshot.DiskId); disk != nil { + diskConfig.Fs = disk.FsFormat + if len(diskConfig.Format) == 0 { + diskConfig.Format = disk.DiskFormat + } + } + diskConfig.SnapshotId = snapshot.Id + diskConfig.DiskType = snapshot.DiskType + diskConfig.SizeMb = snapshot.Size + diskConfig.Backend = storage.StorageType + } + return nil +} + +func fillDiskConfigByImage(ctx context.Context, userCred mcclient.TokenCredential, + diskConfig *SDiskConfig, imageId string) error { + if userCred == nil { + diskConfig.ImageId = imageId + } else { + image, err := CachedimageManager.getImageInfo(ctx, userCred, imageId, false) + if err != nil { + log.Errorf("getImageInfo fail %s", err) + return err + } + if image.Status != IMAGE_STATUS_ACTIVE { + return httperrors.NewInvalidStatusError("Image status is not active") + } + diskConfig.ImageId = image.Id + diskConfig.ImageProperties = image.Properties + if len(diskConfig.Format) == 0 { + diskConfig.Format = image.DiskFormat + } + // diskConfig.ImageDiskFormat = image.DiskFormat + CachedimageManager.ImageAddRefCount(image.Id) + if diskConfig.SizeMb == 0 { + diskConfig.SizeMb = image.MinDisk // MB + } + } + return nil +} + func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, info string) (string, error) { image, err := CachedimageManager.getImageInfo(ctx, userCred, info, false) if err != nil { @@ -1111,6 +1166,10 @@ func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { if len(diskConfig.ImageId) > 0 { self.TemplateId = diskConfig.ImageId self.DiskType = DISK_TYPE_SYS + } else if len(diskConfig.SnapshotId) > 0 { + // XXX: HACK reuse template id as snapshot id + // self.TemplateId = "snapshot-" + diskConfig.SnapshotId + self.DiskType = diskConfig.DiskType } if len(diskConfig.Fs) > 0 { self.FsFormat = diskConfig.Fs diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 352012aa0b..de39aae35e 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -689,6 +689,9 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, httperrors.NewInputParameterError("Invalid root image: %s", err) } + if len(diskConfig.ImageId) > 0 && diskConfig.DiskType != DISK_TYPE_SYS { + return nil, httperrors.NewBadRequestError("Snapshot error: disk index 0 but disk type is %s", diskConfig.DiskType) + } if len(diskConfig.Backend) == 0 { diskConfig.Backend = STORAGE_LOCAL @@ -777,6 +780,9 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, httperrors.NewInputParameterError("parse disk description error %s", err) } + if diskConfig.DiskType == DISK_TYPE_SYS { + return nil, httperrors.NewBadRequestError("Snapshot error: disk index %d > 0 but disk type is %s", idx, DISK_TYPE_SYS) + } if len(diskConfig.Backend) == 0 { diskConfig.Backend = rootStorageType } @@ -2344,6 +2350,10 @@ func (self *SGuest) CreateDisksOnHost(ctx context.Context, userCred mcclient.Tok return err } data.Add(jsonutils.NewString(disk.Id), fmt.Sprintf("disk.%d.id", idx)) + log.Errorln("CreateDisksOnHost XXXXXXXXXX", diskConfig.SnapshotId) + if len(diskConfig.SnapshotId) > 0 { + data.Add(jsonutils.NewString(diskConfig.SnapshotId), fmt.Sprintf("disk.%d.snapshot", idx)) + } } return nil } diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 9fa926df40..8dfdfe95b4 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -629,7 +629,7 @@ func (self *SHost) GetFetchUrl() string { port = 80 } } - return fmt.Sprintf("%s://%s:%d", managerUrl.Scheme, managerUrl.Host, port+40000) + return fmt.Sprintf("%s://%s:%d", managerUrl.Scheme, strings.Split(managerUrl.Host, ":")[0], port+40000) } func (self *SHost) GetAttachedStorages(storageType string) []SStorage { diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 4f35356546..3e369f6527 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -74,6 +74,8 @@ type ComputeOptions struct { NfsDefaultImageCacheDir string `default:"image_cache"` + SnapshotCreateDiskProtocol string `help:"Snapshot create disk protocol" choices:"url|fuse" default:"fuse"` + cloudcommon.DBOptions } diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index 2305667475..264689b650 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -3,10 +3,10 @@ package tasks import ( "context" "fmt" + "time" "yunion.io/x/jsonutils" - "time" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -66,11 +66,9 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d } disk := iDisk.(*models.SDisk) if disk.Status == models.DISK_INIT { - snapInfo, err := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) - if err != nil { - snapInfo = "" - } - err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapInfo, self.GetTaskId()) + snapshotId, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) + log.Errorln("XXXXXXXXXXXXXX", snapshotId) + err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapshotId, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) return diff --git a/pkg/hostimage/host_image_service.go b/pkg/hostimage/host_image_service.go new file mode 100644 index 0000000000..fb8a2846f6 --- /dev/null +++ b/pkg/hostimage/host_image_service.go @@ -0,0 +1,234 @@ +package hostimage + +import ( + "context" + "fmt" + "net/http" + "os" + "path" + "strconv" + "strings" + "time" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/cloudcommon/consts" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" + + "github.com/pierrec/lz4" +) + +type SHostImageOptions struct { + cloudcommon.Options + LocalImagePath []string `help:"Local Image Paths"` + SnapshotDirSuffix string `help:"Snapshot dir name equal diskId concat snapshot dir suffix" default:"_snap"` +} + +var HostImageOptions SHostImageOptions + +func StartService() { + consts.SetServiceType("host-image") + cloudcommon.ParseOptions(&HostImageOptions, &HostImageOptions.Options, os.Args, "host.conf") + HostImageOptions.Port += 40000 + cloudcommon.InitAuth(&HostImageOptions.Options, func() { + log.Infof("Auth complete!!") + }) + app := cloudcommon.InitApp(&HostImageOptions.Options) + initHandlers(app, "") + cloudcommon.ServeForever(app, &HostImageOptions.Options) +} + +func initHandlers(app *appsrv.Application, prefix string) { + app.AddHandler("GET", fmt.Sprintf("%s/disks/", prefix), auth.Authenticate(getImage)) + app.AddHandler("GET", fmt.Sprintf("%s/snapshots//", prefix), auth.Authenticate(getImage)) + app.AddHandler("HEAD", fmt.Sprintf("%s/disks/", prefix), auth.Authenticate(getImageMeta)) + app.AddHandler("HEAD", fmt.Sprintf("%s/snapshots//", prefix), auth.Authenticate(getImageMeta)) +} + +func getDiskPath(diskId string) string { + for _, imagePath := range HostImageOptions.LocalImagePath { + diskPath := path.Join(imagePath, diskId) + if _, err := os.Stat(diskPath); !os.IsNotExist(err) { + return diskPath + } + } + return "" +} + +func getSnapshotPath(diskId, snapshotId string) string { + for _, imagePath := range HostImageOptions.LocalImagePath { + diskPath := path.Join(imagePath, "snapshots", + diskId+HostImageOptions.SnapshotDirSuffix, snapshotId) + if _, err := os.Stat(diskPath); !os.IsNotExist(err) { + return diskPath + } + } + return "" +} + +func inputCheck(ctx context.Context) (string, error) { + var userCred = auth.FetchUserCredential(ctx, nil) + if !userCred.HasSystemAdminPrivelege() { + return "", httperrors.NewForbiddenError("System admin only") + + } + + var params = appctx.AppContextParams(ctx) + var sid = params[""] + var imagePath string + if diskId, ok := params[""]; ok { + imagePath = getSnapshotPath(diskId, sid) + } else { + imagePath = getDiskPath(sid) + } + if len(imagePath) == 0 { + return "", httperrors.NewNotFoundError("Disk not found") + } + return imagePath, nil +} + +func parseRange(reqRange string) (int64, int64, error) { + if !strings.HasPrefix(reqRange, "bytes=") { + return 0, 0, httperrors.NewInputParameterError("Invalid range header") + } + reqRange = reqRange[len("bytes="):] + ranges := strings.Split(reqRange, "-") + if len(ranges) != 2 { + return 0, 0, httperrors.NewInputParameterError("Invalid range header") + } + startPos, err := strconv.ParseInt(ranges[0], 10, 0) + if err != nil { + return 0, 0, httperrors.NewInputParameterError("Invalid range header") + } + endPos, err := strconv.ParseInt(ranges[1], 10, 0) + if err != nil { + return 0, 0, httperrors.NewInputParameterError("Invalid range header") + } + return startPos, endPos, nil +} + +func getImage(ctx context.Context, w http.ResponseWriter, r *http.Request) { + imagePath, err := inputCheck(ctx) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } + + var f IImage + var startPos, endPos int64 + var rateLimit int64 = -1 + + if r.Header.Get("X-Read-File") == "true" { + f = &SFile{} + } else { + f = &SQcow2Image{} + } + if err = f.Open(imagePath, true); err != nil { + log.Errorf("Open image error: %s", err) + httperrors.GeneralServerError(w, err) + return + } + defer f.Close() // Remenber close fd + + endPos = f.Length() - 1 + reqRange := r.Header.Get("Range") + if len(reqRange) > 0 { + startPos, endPos, err = parseRange(reqRange) + if err != nil { + log.Errorf("Parse range error: %s", err) + httperrors.GeneralServerError(w, err) + return + } + } + + strRateLimit := r.Header.Get("X-Rate-Limit-Mbps") + if len(strRateLimit) > 0 { + rateLimit, err = strconv.ParseInt(strRateLimit, 10, 0) + if err != nil { + log.Errorf("Parse ratelimit error: %s", err) + httperrors.InvalidInputError(w, "Invaild rate limit header") + return + } + } + + streamHeader(w, f, startPos, endPos) + startStream(w, f, startPos, endPos, rateLimit) +} + +func streamHeader(w http.ResponseWriter, f IImage, startPos, endPos int64) { + var statusCode = http.StatusOK + w.Header().Set("Content-Type", "application/octet-stream") + if startPos > 0 || endPos < f.Length()-1 { + statusCode = http.StatusPartialContent + w.Header().Set("Content-Range", + fmt.Sprintf("bytes %d-%d/%d", startPos, endPos, f.Length())) + } + w.WriteHeader(statusCode) +} + +func startStream(w http.ResponseWriter, f IImage, startPos, endPos, rateLimit int64) { + var CHUNK_SIZE int64 = 4 * 1024 + var readSize int64 = CHUNK_SIZE + var sendBytes int64 + var lz4Writer = lz4.NewWriter(w) + var startTime = time.Now() + + for startPos < endPos { + if endPos-startPos < CHUNK_SIZE { + readSize = endPos - startPos + 1 + } + buf, total := f.Read(startPos, readSize) + if total < 0 { + log.Errorf("Read image error: %d", total) + goto fail + } + startPos += readSize + wSize, err := lz4Writer.Write(buf) + if err != nil { + log.Errorf("lz4Write error: %s", err) + goto fail + } + sendBytes += int64(wSize) + if rateLimit > 0 { + tmDelta := time.Now().Sub(startTime) + tms := tmDelta.Seconds() + vtmDelta := float64(sendBytes*8) / float64(1024.0*1024.0*rateLimit) + if vtmDelta > tms { + time.Sleep(time.Duration(vtmDelta - tms)) + } + } + } + +fail: + if err := lz4Writer.Close(); err != nil { + log.Errorf("lz4 Close error: %s", err) + } +} + +func getImageMeta(ctx context.Context, w http.ResponseWriter, r *http.Request) { + imagePath, err := inputCheck(ctx) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } + + var f IImage + if r.Header.Get("X-Read-File") == "true" { + f = &SFile{} + } else { + f = &SQcow2Image{} + } + if err = f.Open(imagePath, true); err != nil { + httperrors.GeneralServerError(w, err) + return + } + defer f.Close() // Remenber close fd + + w.Header().Set("Content-Length", fmt.Sprintf("%d", f.Length())) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Accept-Ranges", "bytes") + w.WriteHeader(200) +} diff --git a/pkg/hostimage/image.go b/pkg/hostimage/image.go new file mode 100644 index 0000000000..c5631f8b91 --- /dev/null +++ b/pkg/hostimage/image.go @@ -0,0 +1,134 @@ +package hostimage + +/* +#cgo pkg-config: glib-2.0 zlib +#cgo CFLAGS: -I/home/yunion/rpmbuild/SOURCES/qemu/src -I/home/yunion/rpmbuild/SOURCES/qemu/src/include +#cgo LDFLAGS: -laio -lqemuio -lpthread -L /home/yunion/rpmbuild/SOURCES/qemu/src + +#include "libqemuio.h" +#include "qemu/osdep.h" +*/ +import "C" + +import ( + "fmt" + "io" + "os" + "unsafe" +) + +func init() { + C.qemuio_init() +} + +func ReadQcow2(qemuioBlk *C.struct_QemuioBlk, offset int64, count int64) ([]byte, int64) { + if qemuioBlk == nil || offset < 0 || count < 0 { + return nil, -1 + } + b := make([]byte, count) + var total = C.int64_t(0) + ret := C.read_qcow2(qemuioBlk, unsafe.Pointer(&b[0]), C.int64_t(offset), C.int64_t(count), &total) + if ret < 0 { + return nil, int64(ret) + } else { + return b, int64(total) + } +} + +func OpenQcow2(imagePath string, readonly bool) *C.struct_QemuioBlk { + return C.open_qcow2(C.CString(imagePath), C.bool(readonly)) +} + +func Qcow2GetLenth(qemuioBlk *C.struct_QemuioBlk) int64 { + return int64(C.qcow2_get_length(qemuioBlk)) +} + +func CloseQcow2(qemuioBlk *C.struct_QemuioBlk) { + C.close_qcow2(qemuioBlk) +} + +type IImage interface { + // Open image file and its backing file (if have) + Open(imagePath string, readonly bool) error + + // Close may not really close image file handle, just reudce ref count + Close() + + // If return number < 0 indicate read failed + Read(offset, count int64) ([]byte, int64) + + // Get image file length, not file actual length, it's image virtual size + Length() int64 +} + +type SQcow2Image struct { + fd *C.struct_QemuioBlk +} + +func (img *SQcow2Image) Open(imagePath string, readonly bool) error { + fd := OpenQcow2(imagePath, readonly) + if fd == nil { + return fmt.Errorf("Open image %s failed", imagePath) + } else { + img.fd = fd + return nil + } +} + +func (img *SQcow2Image) Read(offset, count int64) ([]byte, int64) { + return ReadQcow2(img.fd, offset, count) +} + +func (img *SQcow2Image) Close() { + CloseQcow2(img.fd) +} + +func (img *SQcow2Image) Length() int64 { + return Qcow2GetLenth(img.fd) +} + +type SFile struct { + fd *os.File +} + +func (f *SFile) Open(imagePath string, readonly bool) error { + var mode = os.O_RDWR + if readonly { + mode = os.O_RDONLY + } + fd, err := os.OpenFile(imagePath, mode, 0644) + if err != nil { + return err + } else { + f.fd = fd + return nil + } +} + +func (f *SFile) Read(offset, count int64) ([]byte, int64) { + buf := make([]byte, count) + var readCount int64 = 0 + for readCount < count { + cnt, err := f.fd.Read(buf[readCount:]) + readCount += int64(cnt) + if err == io.EOF { + return buf[0:readCount], readCount + } + if err != nil { + return nil, -1 + } + } + return buf, readCount +} + +func (f *SFile) Close() { + f.fd.Close() +} + +func (f *SFile) Length() int64 { + stat, e := f.fd.Stat() + if e != nil { + return -1 + } + return stat.Size() +} diff --git a/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go b/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go index 09cb355365..7d7c92d87f 100644 --- a/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go +++ b/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go @@ -106,7 +106,7 @@ func GetOSProfileFromImageProperties(imgProp map[string]string, hypervisor strin } var imgHypers []string imgHyperStr, ok := imgProp["hypervisor"] - if ok && len(imgHyperStr) > 0 { + if ok { imgHypers = strings.Split(imgHyperStr, ",") } else { imgHypers = []string{} diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index 3a6512d2fb..4a7e253e9f 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -85,14 +85,12 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, er k := c.Name() of := ofields[k] nf := fields[k] - if !gotypes.IsNil(of) { - if c.IsPrimary() && !c.IsZero(of) { // skip update primary key - primaries[k] = of - continue - } else if c.IsKeyIndex() && !c.IsZero(of) { - keyIndexes[k] = of - continue - } + if c.IsPrimary() && !c.IsZero(of) { // skip update primary key + primaries[k] = of + continue + } else if c.IsKeyIndex() && !c.IsZero(of) { + keyIndexes[k] = of + continue } nc, ok := c.(*SIntegerColumn) if ok && nc.IsAutoVersion { From 86b3b38f41362b81994aee7c108fa13fc37ca4b6 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Tue, 11 Dec 2018 20:31:36 +0800 Subject: [PATCH 04/34] add snapshot reference count, fix code --- pkg/appsrv/response.go | 3 +-- pkg/compute/models/disks.go | 9 ++++++--- pkg/compute/models/guest_actions.go | 17 ++++++++++++++++- pkg/compute/models/guestdisks.go | 6 ++++++ pkg/compute/models/guests.go | 1 - pkg/compute/models/snapshots.go | 17 +++++++++++++++++ pkg/compute/tasks/disk_delete_task.go | 3 +++ pkg/compute/tasks/guest_create_disk_task.go | 1 - pkg/hostimage/host_image_service.go | 1 - 9 files changed, 49 insertions(+), 9 deletions(-) diff --git a/pkg/appsrv/response.go b/pkg/appsrv/response.go index ac611719af..89e64a49ce 100644 --- a/pkg/appsrv/response.go +++ b/pkg/appsrv/response.go @@ -2,9 +2,8 @@ package appsrv import ( "context" - "net/http" - "fmt" + "net/http" "yunion.io/x/onecloud/pkg/httperrors" ) diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index f80c298bd5..a95aae1ff8 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -100,6 +100,9 @@ type SDisk struct { // # backing template id and type TemplateId string `width:"256" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=True) + // backing snapshot id + SnapshotId string `width:"256" charset:"ascii" nullable:"true" list:"user"` + // # file system FsFormat string `width:"32" charset:"ascii" nullable:"true" list:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=True) // # disk type, OS, SWAP, DAT @@ -446,6 +449,8 @@ func (self *SDisk) StartAllocate(ctx context.Context, host *SHost, storage *SSto content.Add(jsonutils.NewInt(int64(self.DiskSize)), "size") if len(snapshot) > 0 { content.Add(jsonutils.NewString(snapshot), "snapshot") + SnapshotManager.AddRefCount(self.SnapshotId, 1) + self.SetMetadata(ctx, "merge_snapshot", jsonutils.JSONTrue, userCred) } else if len(templateId) > 0 { content.Add(jsonutils.NewString(templateId), "image_id") } @@ -1078,7 +1083,6 @@ func parseDiskInfo(ctx context.Context, userCred mcclient.TokenCredential, info } else if strings.HasPrefix(p, "snapshot-") { // HACK: use snapshot creat disk format snapshot-id // example: snapshot-3140cecb-ccc4-4865-abae-3a5ba8c69d9b - log.Errorln("The snapshot XXXXXXX Create disk", p[len("snapshot-"):]) if err := fillDiskConfigBySnapshot(userCred, &diskConfig, p[len("snapshot-"):]); err != nil { return nil, err } @@ -1167,8 +1171,7 @@ func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { self.TemplateId = diskConfig.ImageId self.DiskType = DISK_TYPE_SYS } else if len(diskConfig.SnapshotId) > 0 { - // XXX: HACK reuse template id as snapshot id - // self.TemplateId = "snapshot-" + diskConfig.SnapshotId + self.SnapshotId = diskConfig.SnapshotId self.DiskType = diskConfig.DiskType } if len(diskConfig.Fs) > 0 { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 19f0397e3e..dac61120c5 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -8,6 +8,7 @@ import ( "net/http" "strconv" "strings" + "time" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -22,7 +23,6 @@ import ( "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/seclib2" - "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/util/billing" @@ -2112,3 +2112,18 @@ func (self *SGuest) doSaveRenewInfo(userCred mcclient.TokenCredential, bc *billi db.OpsLog.LogEvent(self, db.ACT_RENEW, self.GetShortDesc(), userCred) return nil } + +func (self *SGuest) AllowPerformStreamDisksComplete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return db.IsAdminAllowPerform(userCred, self, "stream-disks-complete") +} + +func (self *SGuest) PerformStreamDisksComplete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + for _, disk := range self.GetDisks() { + d := disk.GetDisk() + if len(d.SnapshotId) > 0 { + SnapshotManager.AddRefCount(d.SnapshotId, -1) + d.SetMetadata(ctx, "merge_snapshot", jsonutils.JSONFalse, userCred) + } + } + return nil, nil +} diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index ab86de0496..e6000e0917 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -166,6 +166,12 @@ func (self *SGuestdisk) GetJsonDescAtHost(host *SHost) jsonutils.JSONObject { if len(tid) > 0 { desc.Add(jsonutils.NewString(tid), "template_id") } + if len(disk.SnapshotId) > 0 { + needMerge := disk.GetMetadata("merge_snapshot", nil) + if needMerge == "true" { + desc.Set("merge_snapshot", jsonutils.JSONTrue) + } + } fs := disk.GetFsFormat() if len(fs) > 0 { desc.Add(jsonutils.NewString(fs), "fs") diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index de39aae35e..1420bcc813 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2350,7 +2350,6 @@ func (self *SGuest) CreateDisksOnHost(ctx context.Context, userCred mcclient.Tok return err } data.Add(jsonutils.NewString(disk.Id), fmt.Sprintf("disk.%d.id", idx)) - log.Errorln("CreateDisksOnHost XXXXXXXXXX", diskConfig.SnapshotId) if len(diskConfig.SnapshotId) > 0 { data.Add(jsonutils.NewString(diskConfig.SnapshotId), fmt.Sprintf("disk.%d.snapshot", idx)) } diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index ed9c81a6c4..2a68fdc766 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -49,6 +49,9 @@ type SSnapshot struct { FakeDeleted bool `nullable:"false" default:"false" index:"true"` DiskType string `width:"32" charset:"ascii" nullable:"true" list:"user"` + // create disk from snapshot, snapshot as disk backing file + RefCount int `nullable:"false" default:"0" list:"user"` + CloudregionId string `width:"36" charset:"ascii" nullable:"true" list:"user"` } @@ -246,6 +249,17 @@ func (self *SSnapshot) GetHost() *SHost { return storage.GetMasterHost() } +func (self *SSnapshotManager) AddRefCount(snapshotId string, count int) { + iSnapshot, _ := self.FetchById(snapshotId) + snapshot := iSnapshot.(*SSnapshot) + if snapshot != nil { + self.TableSpec().Update(snapshot, func() error { + snapshot.RefCount += count + return nil + }) + } +} + func (self *SSnapshotManager) GetDiskSnapshotsByCreate(diskId, createdBy string) []SSnapshot { dest := make([]SSnapshot, 0) q := self.Query().SubQuery() @@ -343,6 +357,9 @@ func (self *SSnapshot) StartSnapshotDeleteTask(ctx context.Context, userCred mcc } func (self *SSnapshot) ValidateDeleteCondition(ctx context.Context) error { + if self.RefCount > 0 { + return fmt.Errorf("Snapshot reference(by disk) count > 0, can not delete") + } return nil } diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index 3eac8a0371..ea433d2204 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -93,6 +93,9 @@ func (self *DiskDeleteTask) OnGuestDiskDeleteComplete(ctx context.Context, obj d disk := obj.(*models.SDisk) self.CleanHostSchedCache(disk) db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, disk.GetShortDesc(), self.UserCred) + if len(disk.SnapshotId) > 0 && disk.GetMetadata("merge_snapshot", nil) == "true" { + models.SnapshotManager.AddRefCount(disk.SnapshotId, -1) + } disk.RealDelete(ctx, self.UserCred) self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/guest_create_disk_task.go b/pkg/compute/tasks/guest_create_disk_task.go index 264689b650..2b9181df40 100644 --- a/pkg/compute/tasks/guest_create_disk_task.go +++ b/pkg/compute/tasks/guest_create_disk_task.go @@ -67,7 +67,6 @@ func (self *KVMGuestCreateDiskTask) OnKvmDiskPrepared(ctx context.Context, obj d disk := iDisk.(*models.SDisk) if disk.Status == models.DISK_INIT { snapshotId, _ := self.Params.GetString(fmt.Sprintf("disk.%d.snapshot", diskIndex)) - log.Errorln("XXXXXXXXXXXXXX", snapshotId) err = disk.StartDiskCreateTask(ctx, self.UserCred, false, snapshotId, self.GetTaskId()) if err != nil { self.SetStageFailed(ctx, err.Error()) diff --git a/pkg/hostimage/host_image_service.go b/pkg/hostimage/host_image_service.go index fb8a2846f6..1b04b596be 100644 --- a/pkg/hostimage/host_image_service.go +++ b/pkg/hostimage/host_image_service.go @@ -73,7 +73,6 @@ func inputCheck(ctx context.Context) (string, error) { var userCred = auth.FetchUserCredential(ctx, nil) if !userCred.HasSystemAdminPrivelege() { return "", httperrors.NewForbiddenError("System admin only") - } var params = appctx.AppContextParams(ctx) From b8f3e6632dfed5724ed9551dc55e6c242ea62931 Mon Sep 17 00:00:00 2001 From: TangBin Date: Tue, 11 Dec 2018 20:55:21 +0800 Subject: [PATCH 05/34] update skus climc --- cmd/climc/shell/skus.go | 8 ++-- pkg/compute/models/skus.go | 84 ++++++++++++-------------------------- 2 files changed, 29 insertions(+), 63 deletions(-) diff --git a/cmd/climc/shell/skus.go b/cmd/climc/shell/skus.go index 28d7e56950..61e5392f87 100644 --- a/cmd/climc/shell/skus.go +++ b/cmd/climc/shell/skus.go @@ -11,7 +11,7 @@ import ( func init() { type ServerSkusListOptions struct { options.BaseListOptions - Provider string `help:"provider" choices:"all|kvm|esxi|xen|hyperv|aliyun|azure|aws|qcloud|huawei"` + Provider string `help:"provider" choices:"all|aliyun|azure|aws|qcloud|huawei" default:""` Region string `help:"region Id or name"` Zone string `help:"zone Id or name"` Cpu int `help:"Cpu core count"` @@ -50,9 +50,8 @@ func init() { }) type ServerSkusCreateOptions struct { - CpuCoreCount int `help:"Cpu Count" required:"true" positional:"true"` - MemorySizeMB int `help:"Memory MB" required:"true" positional:"true"` - Provider string `help:"Provider name" choices:"all|kvm|esxi"` + CpuCoreCount int `help:"Cpu Count" required:"true" positional:"true"` + MemorySizeMB int `help:"Memory MB" required:"true" positional:"true"` OsName *string `help:"OS name/type" choices:"Linux|Windows|Any" default:"Any"` InstanceTypeCategory *string `help:"instance type category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"` @@ -96,7 +95,6 @@ func init() { CpuCoreCount *int `help:"Cpu Count"` MemorySizeMB *int `help:"Memory MB"` - Provider string `help:"Provider name" choices:"all|kvm|esxi"` InstanceTypeCategory *string `help:"instance type category" choices:"general_purpose|compute_optimized|memory_optimized|storage_optimized|hardware_accelerated|high_memory|high_storage"` SysDiskResizable *bool `help:"system disk is resizable"` diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index e463b6bfd4..ea5ec58b24 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -66,10 +66,10 @@ type SServerSku struct { CpuCoreCount int `nullable:"false" list:"user" create:"admin_required" update:"admin"` MemorySizeMB int `nullable:"false" list:"user" create:"admin_required" update:"admin"` - OsName string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_required" update:"admin" default:"Any"` // Windows|Linux|Any + OsName string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin" default:"Any"` // Windows|Linux|Any SysDiskResizable bool `default:"true" nullable:"false" list:"user" create:"admin_optional" update:"admin"` - SysDiskType string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_required" update:"admin"` + SysDiskType string `width:"32" charset:"ascii" nullable:"false" list:"user" create:"admin_optional" update:"admin"` SysDiskMinSizeGB int `nullable:"false" list:"user" create:"admin_optional" update:"admin"` // not required。 windows比较新的版本都是50G左右。 SysDiskMaxSizeGB int `nullable:"false" list:"user" create:"admin_optional" update:"admin"` // not required @@ -94,24 +94,20 @@ type SServerSku struct { } func inWhiteList(provider string) bool { - // 只有为true的hypervisor才进行创建和更新操作 + // provider 字段为空时表示私有云套餐 if len(provider) == 0 { return true - } - switch provider { - case HYPERVISOR_ESXI, HYPERVISOR_KVM, "all": // 空或者all时。表示`通用`私用云instance type列表 - return true - default: + } else { return false } } func genInstanceType(family string, cpu, mem_mb int64) (string, error) { - if cpu < 0 { + if cpu <= 0 { return "", fmt.Errorf("cpu_core_count should great than zero") } - if mem_mb < 0 || mem_mb%1024 != 0 { + if mem_mb <= 0 || mem_mb%1024 != 0 { return "", fmt.Errorf("memory_size_mb should great than zero. and should be integral multiple of 1024") } @@ -143,9 +139,7 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, return nil, httperrors.NewForbiddenError("can not create instance_type for public cloud %s", provider) } - if provider == "all" { - data.Remove("provider") - } + data.Remove("provider") regionStr := jsonutils.GetAnyString(data, []string{"region", "region_id", "cloudregion", "cloudregion_id"}) if len(regionStr) > 0 { @@ -189,7 +183,7 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, if err != nil { return nil, httperrors.NewInputParameterError("memory_size_mb should not be empty") } else { - data.Set("cpu_core_count", jsonutils.NewInt(mem)) + data.Set("memory_size_mb", jsonutils.NewInt(mem)) } category, _ := data.GetString("instance_type_category") @@ -207,15 +201,11 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, data.Set("name", jsonutils.NewString(name)) - q := self.Query().Equals("name", name) - if len(provider) > 0 && provider != "all" { - q = q.Equals("provider", provider) - } else { - q = q.Filter(sqlchemy.OR( - sqlchemy.IsNull(q.Field("provider")), - sqlchemy.IsEmpty(q.Field("provider")), - )) - } + q := self.Query() + q = q.Equals("name", name).Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + )) if q.Count() > 0 { return nil, httperrors.NewDuplicateResourceError("Duplicate sku %s", name) @@ -348,10 +338,7 @@ func (self *SServerSku) ValidateUpdateData( if err == nil && !inWhiteList(provider) { return nil, httperrors.NewForbiddenError("can not update instance_type for public cloud %s", provider) } - - if provider == "all" { - data.Remove("provider") - } + data.Remove("provider") zoneStr := jsonutils.GetAnyString(data, []string{"zone", "zone_id"}) if len(zoneStr) > 0 { @@ -405,15 +392,11 @@ func (self *SServerSku) ValidateUpdateData( data.Set("name", jsonutils.NewString(name)) - q := self.GetModelManager().Query().Equals("name", name) - if len(provider) > 0 && provider != "all" { - q = q.Equals("provider", provider) - } else { - q = q.Filter(sqlchemy.OR( - sqlchemy.IsNull(q.Field("provider")), - sqlchemy.IsEmpty(q.Field("provider")), - )) - } + q := self.GetModelManager().Query() + q = q.Equals("name", name).Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + )) if q.Count() > 0 { return nil, httperrors.NewDuplicateResourceError("sku cpu %d mem %d(Mb) already exists", cpu, mem) @@ -449,28 +432,13 @@ func (self *SServerSku) GetZoneExternalId() (string, error) { func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { provider := jsonutils.GetAnyString(query, []string{"provider"}) - if inWhiteList(provider) { - // provider 参数为空或者all时。表示查询`通用`私用云instance type列表 - if provider == "" { - q = q.Filter(sqlchemy.OR( - sqlchemy.IsNull(q.Field("provider")), - sqlchemy.IsEmpty(q.Field("provider")), - sqlchemy.Equals(q.Field("provider"), HYPERVISOR_KVM), - sqlchemy.Equals(q.Field("provider"), HYPERVISOR_ESXI), - )) - } else if provider == "all" { - q = q.Filter(sqlchemy.OR( - sqlchemy.IsNull(q.Field("provider")), - sqlchemy.IsEmpty(q.Field("provider")), - )) - } else { - q = q.Filter(sqlchemy.OR( - sqlchemy.IsNull(q.Field("provider")), - sqlchemy.IsEmpty(q.Field("provider")), - sqlchemy.Equals(q.Field("provider"), provider), - )) - } - } else { + // provider 参数为all时。表示查询所有instance type + if provider == "" { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + )) + } else if provider != "all" { q = q.Equals("provider", provider) } From 5d5b9c8c85416adf86b1bffa2a385cbdfa042657 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 00:09:07 +0800 Subject: [PATCH 06/34] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=EF=BC=9A1.=20region?= =?UTF-8?q?=E7=9A=84=E5=8C=BA=E5=9F=9F=E4=BF=A1=E6=81=AF=202.=20=E5=AE=8C?= =?UTF-8?q?=E5=96=84host=E7=9A=84=E5=85=AC=E6=9C=89=E4=BA=91=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/cloudregions.go | 12 ++-- cmd/climc/shell/hosts.go | 21 ++++-- pkg/cloudprovider/fakeregion.go | 8 +-- pkg/cloudprovider/geoinfo.go | 9 +++ pkg/cloudprovider/resources.go | 5 +- pkg/compute/models/billingresource.go | 44 ++++++------- pkg/compute/models/cloudregions.go | 13 ++-- pkg/compute/models/hosts.go | 82 ++++++++++++++++++++---- pkg/mcclient/modules/mod_cloudregions.go | 6 +- pkg/util/aliyun/latitud_and_longitude.go | 42 ++++++------ pkg/util/aliyun/region.go | 15 ++--- pkg/util/aws/latitude_and_longitude.go | 42 ++++++------ pkg/util/aws/region.go | 27 ++------ pkg/util/azure/region.go | 11 ++-- pkg/util/qcloud/latitud_and_longitude.go | 42 ++++++------ pkg/util/qcloud/region.go | 13 +--- 16 files changed, 221 insertions(+), 171 deletions(-) create mode 100644 pkg/cloudprovider/geoinfo.go diff --git a/cmd/climc/shell/cloudregions.go b/cmd/climc/shell/cloudregions.go index 48223281d7..6084aa381b 100644 --- a/cmd/climc/shell/cloudregions.go +++ b/cmd/climc/shell/cloudregions.go @@ -11,10 +11,11 @@ import ( func init() { type CloudregionListOptions struct { options.BaseListOptions - Private bool `help:"show private cloud regions only"` - Public bool `help:"show public cloud regions only"` - Manager string `help:"Show regions belongs to the cloud provider"` - Usable bool `help:"List regions that are usable"` + Private bool `help:"show private cloud regions only"` + Public bool `help:"show public cloud regions only"` + Manager string `help:"Show regions belongs to the cloud provider"` + Provider string `help:"List regions of the public cloud provider" choices:"Aliyun|Qcloud|Azure|Aws|Huawei"` + Usable bool `help:"List regions that are usable"` } R(&CloudregionListOptions{}, "cloud-region-list", "List cloud regions", func(s *mcclient.ClientSession, args *CloudregionListOptions) error { var params *jsonutils.JSONDict @@ -38,6 +39,9 @@ func init() { if len(args.Manager) > 0 { params.Add(jsonutils.NewString(args.Manager), "manager") } + if len(args.Provider) > 0 { + params.Add(jsonutils.NewString(args.Provider), "provider") + } result, err := modules.Cloudregions.List(s, params) if err != nil { return err diff --git a/cmd/climc/shell/hosts.go b/cmd/climc/shell/hosts.go index 8e468f9ccd..ff0aebb5b0 100644 --- a/cmd/climc/shell/hosts.go +++ b/cmd/climc/shell/hosts.go @@ -13,8 +13,8 @@ func init() { type HostListOptions struct { Schedtag string `help:"List hosts in schedtag"` Zone string `help:"List hosts in zone"` + Region string `help:"List hosts in region"` Wire string `help:"List hosts in wire"` - VCenter string `help:"List hosts in vcenter"` Image string `help:"List hosts cached images"` Storage string `help:"List hosts attached to storages"` Baremetal string `help:"List hosts that is managed by baremetal system" choices:"true|false"` @@ -27,8 +27,11 @@ func init() { ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` - Manager string `help:"Show regions belongs to the cloud provider"` - Usable bool `help:"List all zones that is usable"` + Manager string `help:"List hosts belongs to the cloud provider"` + Account string `help:"List hosts belongs to the cloud account"` + Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` + + Usable bool `help:"List all zones that is usable"` options.BaseListOptions } @@ -48,12 +51,12 @@ func init() { if len(args.Zone) > 0 { params.Add(jsonutils.NewString(args.Zone), "zone") } + if len(args.Region) > 0 { + params.Add(jsonutils.NewString(args.Region), "region") + } if len(args.Wire) > 0 { params.Add(jsonutils.NewString(args.Wire), "wire") } - if len(args.VCenter) > 0 { - params.Add(jsonutils.NewString(args.VCenter), "vcenter") - } if len(args.Image) > 0 { params.Add(jsonutils.NewString(args.Image), "cachedimage") } @@ -73,6 +76,12 @@ func init() { if len(args.Manager) > 0 { params.Add(jsonutils.NewString(args.Manager), "manager") } + if len(args.Account) > 0 { + params.Add(jsonutils.NewString(args.Account), "account") + } + if len(args.Provider) > 0 { + params.Add(jsonutils.NewString(args.Provider), "provider") + } if args.Usable { params.Add(jsonutils.JSONTrue, "usable") diff --git a/pkg/cloudprovider/fakeregion.go b/pkg/cloudprovider/fakeregion.go index 0634a97bd9..d2193bfe4c 100644 --- a/pkg/cloudprovider/fakeregion.go +++ b/pkg/cloudprovider/fakeregion.go @@ -36,12 +36,8 @@ func (region *SFakeOnPremiseRegion) GetMetadata() *jsonutils.JSONDict { return nil } -func (region *SFakeOnPremiseRegion) GetLatitude() float32 { - return 0.0 -} - -func (region *SFakeOnPremiseRegion) GetLongitude() float32 { - return 0.0 +func (region *SFakeOnPremiseRegion) GetGeographicInfo() SGeographicInfo { + return SGeographicInfo{} } func (region *SFakeOnPremiseRegion) GetIZones() ([]ICloudZone, error) { diff --git a/pkg/cloudprovider/geoinfo.go b/pkg/cloudprovider/geoinfo.go new file mode 100644 index 0000000000..9d7dabe250 --- /dev/null +++ b/pkg/cloudprovider/geoinfo.go @@ -0,0 +1,9 @@ +package cloudprovider + +type SGeographicInfo struct { + Latitude float32 `list:"user" update:"admin" create:"admin_optional"` + Longitude float32 `list:"user" update:"admin" create:"admin_optional"` + + City string `list:"user" width:"32" update:"admin" create:"admin_optional"` + CountryCode string `list:"user" width:"4" update:"admin" create:"admin_optional"` +} diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index b05533277e..26430467f3 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -33,8 +33,9 @@ type IBillingResource interface { type ICloudRegion interface { ICloudResource - GetLatitude() float32 - GetLongitude() float32 + // GetLatitude() float32 + // GetLongitude() float32 + GetGeographicInfo() SGeographicInfo GetIZones() ([]ICloudZone, error) GetIVpcs() ([]ICloudVpc, error) diff --git a/pkg/compute/models/billingresource.go b/pkg/compute/models/billingresource.go index 218aef12c4..69c575a207 100644 --- a/pkg/compute/models/billingresource.go +++ b/pkg/compute/models/billingresource.go @@ -45,24 +45,24 @@ func (self *SBillingResourceBase) IsValidPrePaid() bool { } type SCloudBillingInfo struct { - Provider string - Account string - AccountId string - SubAccount string - SubAccountId string - SubAccountProject string - SubAccountProjectId string - Region string - RegionId string - RegionExtId string - Zone string - ZoneId string - ZoneExtId string - PriceKey string - ChargeType string - InternetChargeType string - ExpiredAt time.Time - BillingCycle string + Provider string `json:",omitempty"` + Account string `json:",omitempty"` + AccountId string `json:",omitempty"` + Manager string `json:",omitempty"` + ManagerId string `json:",omitempty"` + ManagerProject string `json:",omitempty"` + ManagerProjectId string `json:",omitempty"` + Region string `json:",omitempty"` + RegionId string `json:",omitempty"` + RegionExtId string `json:",omitempty"` + Zone string `json:",omitempty"` + ZoneId string `json:",omitempty"` + ZoneExtId string `json:",omitempty"` + PriceKey string `json:",omitempty"` + ChargeType string `json:",omitempty"` + InternetChargeType string `json:",omitempty"` + ExpiredAt time.Time `json:",omitempty"` + BillingCycle string `json:",omitempty"` } func MakeCloudBillingInfo(region *SCloudregion, zone *SZone, provider *SCloudprovider) SCloudBillingInfo { @@ -79,14 +79,14 @@ func MakeCloudBillingInfo(region *SCloudregion, zone *SZone, provider *SCloudpro } if provider != nil { - info.SubAccount = provider.GetName() - info.SubAccountId = provider.GetId() + info.Manager = provider.GetName() + info.ManagerId = provider.GetId() if len(provider.ProjectId) > 0 { - info.SubAccountProjectId = provider.ProjectId + info.ManagerProjectId = provider.ProjectId tc, err := db.TenantCacheManager.FetchTenantById(appctx.Background, provider.ProjectId) if err == nil { - info.SubAccountProject = tc.GetName() + info.ManagerProject = tc.GetName() } } diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 8d3c327fc7..6d04dbd52c 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -43,9 +43,9 @@ func init() { type SCloudregion struct { db.SEnabledStatusStandaloneResourceBase - Latitude float32 `list:"user"` - Longitude float32 `list:"user"` - Provider string `width:"64" charset:"ascii" list:"user"` + cloudprovider.SGeographicInfo + + Provider string `width:"64" charset:"ascii" list:"user"` } func (manager *SCloudregionManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { @@ -227,8 +227,7 @@ func (self *SCloudregion) syncWithCloudRegion(cloudRegion cloudprovider.ICloudRe _, err := self.GetModelManager().TableSpec().Update(self, func() error { self.Name = cloudRegion.GetName() self.Status = cloudRegion.GetStatus() - self.Latitude = cloudRegion.GetLatitude() - self.Longitude = cloudRegion.GetLongitude() + self.SGeographicInfo = cloudRegion.GetGeographicInfo() self.Provider = cloudRegion.GetProvider() self.IsEmulated = cloudRegion.IsEmulated() @@ -247,8 +246,7 @@ func (manager *SCloudregionManager) newFromCloudRegion(cloudRegion cloudprovider region.ExternalId = cloudRegion.GetGlobalId() region.Name = cloudRegion.GetName() - region.Latitude = cloudRegion.GetLatitude() - region.Longitude = cloudRegion.GetLongitude() + region.SGeographicInfo = cloudRegion.GetGeographicInfo() region.Status = cloudRegion.GetStatus() region.Enabled = true region.Provider = cloudRegion.GetProvider() @@ -352,6 +350,7 @@ func (manager *SCloudregionManager) ListItemFilter(ctx context.Context, q *sqlch } q = q.Equals("provider", manager.Provider) } + if jsonutils.QueryBoolean(query, "usable", false) { networks := NetworkManager.Query().SubQuery() wires := WireManager.Query().SubQuery() diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 67eb56f7ce..4c3b2fda21 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -195,9 +195,10 @@ func (self *SHost) AllowDeleteItem(ctx context.Context, userCred mcclient.TokenC } func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + queryDict := query.(*jsonutils.JSONDict) + resType, _ := query.GetString("resource_type") if len(resType) > 0 { - queryDict := query.(*jsonutils.JSONDict) queryDict.Remove("resource_type") switch resType { @@ -267,23 +268,65 @@ func (manager *SHostManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) if len(zoneStr) > 0 { - zone, _ := ZoneManager.FetchByIdOrName(nil, zoneStr) - if zone == nil { - return nil, httperrors.NewResourceNotFoundError("Zone %s not found", zoneStr) + zone, err := ZoneManager.FetchByIdOrName(nil, zoneStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), zoneStr) + } + return nil, httperrors.NewGeneralError(err) } q = q.Filter(sqlchemy.Equals(q.Field("zone_id"), zone.GetId())) + + queryDict.Remove("zone_id") } + + regionStr := jsonutils.GetAnyString(query, []string{"region", "region_id"}) + if len(regionStr) > 0 { + region, err := CloudregionManager.FetchByIdOrName(nil, regionStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionStr) + } + return nil, httperrors.NewGeneralError(err) + } + subq := ZoneManager.Query("id").Equals("cloudregion_id", region.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("zone_id"), subq)) + } + // vcenter // zone // cachedimage - managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) + managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"}) if len(managerStr) > 0 { - provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr) - if provider == nil { - return nil, httperrors.NewResourceNotFoundError("provider %s not found", managerStr) + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr) + } + return nil, httperrors.NewGeneralError(err) } q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId())) + queryDict.Remove("manager_id") + } + + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) } usable := jsonutils.QueryBoolean(query, "usable", false) @@ -1941,13 +1984,25 @@ func (self *SHost) getGuestsResource(status string) *SHostGuestResourceUsage { } func (self *SHost) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict) *jsonutils.JSONDict { - zone := self.GetZone() + /*zone := self.GetZone() if zone != nil { extra.Add(jsonutils.NewString(zone.Id), "zone_id") extra.Add(jsonutils.NewString(zone.Name), "zone") - extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region") - extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id") - } + if len(zone.ExternalId) > 0 { + extra.Add(jsonutils.NewString(zone.ExternalId), "") + } + region := zone.GetRegion() + if region != nil { + extra.Add(jsonutils.NewString(zone.GetRegion().GetName()), "region") + extra.Add(jsonutils.NewString(zone.GetRegion().GetId()), "region_id") + } + }*/ + + info := self.getCloudBillingInfo() + infoJson := jsonutils.Marshal(&info) + log.Debugf("%s", infoJson.String()) + extra.Update(infoJson) + server := self.GetBaremetalServer() if server != nil { extra.Add(jsonutils.NewString(server.Id), "server_id") @@ -2000,7 +2055,8 @@ func (self *SHost) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict } extra.Add(jsonutils.NewFloat(memCommitRate), "mem_commit_rate") extra.Add(self.GetHardwareSpecification(), "spec") - extra = self.SManagedResourceBase.getExtraDetails(ctx, extra) + + // extra = self.SManagedResourceBase.getExtraDetails(ctx, extra) if self.IsPrepaidRecycle() { extra.Add(jsonutils.JSONTrue, "is_prepaid_recycle") diff --git a/pkg/mcclient/modules/mod_cloudregions.go b/pkg/mcclient/modules/mod_cloudregions.go index ae41321e22..a2aad96493 100644 --- a/pkg/mcclient/modules/mod_cloudregions.go +++ b/pkg/mcclient/modules/mod_cloudregions.go @@ -6,8 +6,10 @@ var ( func init() { Cloudregions = NewComputeManager("cloudregion", "cloudregions", - []string{"ID", "Name", "Enabled", "Status", "Provider", "Latitude", "Longitude", - "vpc_count", "zone_count", "guest_count", "guest_increment_count"}, + []string{"ID", "Name", "Enabled", "Status", "Provider", + "Latitude", "Longitude", "City", "Country_Code", + "vpc_count", "zone_count", "guest_count", "guest_increment_count", + "External_Id"}, []string{}) registerCompute(&Cloudregions) diff --git a/pkg/util/aliyun/latitud_and_longitude.go b/pkg/util/aliyun/latitud_and_longitude.go index fac2ab850d..7329667fb3 100644 --- a/pkg/util/aliyun/latitud_and_longitude.go +++ b/pkg/util/aliyun/latitud_and_longitude.go @@ -1,23 +1,25 @@ package aliyun -var LatitudeAndLongitude = map[string]map[string]float32{ - "cn-qingdao": {"latitude": 36.067108, "longitude": 120.382607}, - "cn-beijing": {"latitude": 39.904202, "longitude": 116.407394}, - "cn-zhangjiakou": {"latitude": 40.767544, "longitude": 114.886337}, - "cn-huhehaote": {"latitude": 40.842358, "longitude": 111.749992}, - "cn-hangzhou": {"latitude": 30.274084, "longitude": 120.155067}, - "cn-shanghai": {"latitude": 31.230391, "longitude": 121.473701}, - "cn-shenzhen": {"latitude": 22.543097, "longitude": 114.057861}, - "cn-hongkong": {"latitude": 22.396427, "longitude": 114.109497}, - "ap-northeast-1": {"latitude": 35.709026, "longitude": 139.731995}, - "ap-southeast-1": {"latitude": 1.352083, "longitude": 103.819839}, - "ap-southeast-2": {"latitude": -33.868820, "longitude": 151.209290}, - "ap-southeast-3": {"latitude": 3.139003, "longitude": 101.686852}, - "ap-southeast-5": {"latitude": -6.175110, "longitude": 106.865036}, - "ap-south-1": {"latitude": 19.075983, "longitude": 72.877655}, - "us-east-1": {"latitude": 37.431572, "longitude": -78.656891}, - "us-west-1": {"latitude": 37.387474, "longitude": -122.057541}, - "eu-west-1": {"latitude": 51.507351, "longitude": -0.127758}, - "me-east-1": {"latitude": 25.204849, "longitude": 55.270782}, - "eu-central-1": {"latitude": 50.110924, "longitude": 8.682127}, +import "yunion.io/x/onecloud/pkg/cloudprovider" + +var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{ + "cn-qingdao": {Latitude: 36.067108, Longitude: 120.382607, City: "Qingdao", CountryCode: "CN"}, + "cn-beijing": {Latitude: 39.904202, Longitude: 116.407394, City: "Beijing", CountryCode: "CN"}, + "cn-zhangjiakou": {Latitude: 40.767544, Longitude: 114.886337, City: "Zhangjiakou", CountryCode: "CN"}, + "cn-huhehaote": {Latitude: 40.842358, Longitude: 111.749992, City: "Huhehaote", CountryCode: "CN"}, + "cn-hangzhou": {Latitude: 30.274084, Longitude: 120.155067, City: "Hangzhou", CountryCode: "CN"}, + "cn-shanghai": {Latitude: 31.230391, Longitude: 121.473701, City: "Shanghai", CountryCode: "CN"}, + "cn-shenzhen": {Latitude: 22.543097, Longitude: 114.057861, City: "Shenzhen", CountryCode: "CN"}, + "cn-hongkong": {Latitude: 22.396427, Longitude: 114.109497, City: "Hongkong", CountryCode: "CN"}, + "ap-northeast-1": {Latitude: 35.709026, Longitude: 139.731995, City: "Tokyo", CountryCode: "JP"}, + "ap-southeast-1": {Latitude: 1.352083, Longitude: 103.819839, City: "Singapore", CountryCode: "SG"}, + "ap-southeast-2": {Latitude: -33.868820, Longitude: 151.209290, City: "Sydney", CountryCode: "AU"}, + "ap-southeast-3": {Latitude: 3.139003, Longitude: 101.686852, City: "Kuala Lumpur", CountryCode: "MY"}, + "ap-southeast-5": {Latitude: -6.175110, Longitude: 106.865036, City: "Jakarta", CountryCode: "ID"}, + "ap-south-1": {Latitude: 19.075983, Longitude: 72.877655, City: "Mumbai", CountryCode: "IN"}, + "us-east-1": {Latitude: 37.431572, Longitude: -78.656891, City: "Virgina", CountryCode: "US"}, + "us-west-1": {Latitude: 37.387474, Longitude: -122.057541, City: "Siliconvalley", CountryCode: "US"}, + "eu-west-1": {Latitude: 51.507351, Longitude: -0.127758, City: "London", CountryCode: "GB"}, + "me-east-1": {Latitude: 25.204849, Longitude: 55.270782, City: "Dubai", CountryCode: "AE"}, + "eu-central-1": {Latitude: 50.110924, Longitude: 8.682127, City: "Frankfurt", CountryCode: "DE"}, } diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index 015cc2d0d9..18b176bb1e 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -108,18 +108,11 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_ALIYUN } -func (self *SRegion) GetLatitude() float32 { - if locationInfo, ok := LatitudeAndLongitude[self.RegionId]; ok { - return locationInfo["latitude"] +func (self *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { + if info, ok := LatitudeAndLongitude[self.RegionId]; ok { + return info } - return 0.0 -} - -func (self *SRegion) GetLongitude() float32 { - if locationInfo, ok := LatitudeAndLongitude[self.RegionId]; ok { - return locationInfo["longitude"] - } - return 0.0 + return cloudprovider.SGeographicInfo{} } func (self *SRegion) GetStatus() string { diff --git a/pkg/util/aws/latitude_and_longitude.go b/pkg/util/aws/latitude_and_longitude.go index f3082a9e6c..30ab4cb767 100644 --- a/pkg/util/aws/latitude_and_longitude.go +++ b/pkg/util/aws/latitude_and_longitude.go @@ -1,23 +1,25 @@ package aws -var LatitudeAndLongitude = map[string]map[string]float32{ - "ap-south-1": {"latitude": 19.0759837, "longitude": 72.8776559}, - "ap-northeast-3": {"latitude": 34.6937378, "longitude": 135.5021651}, - "us-east-1": {"latitude": 37.4315734, "longitude": -78.6568942}, - "us-east-2": {"latitude": 40.4172871, "longitude": -82.90712300000001}, - "ap-southeast-2": {"latitude": -33.8688197, "longitude": 151.2092955}, - "cn-northwest-1": {"latitude": 37.198731, "longitude": 106.1580937}, - "eu-west-1": {"latitude": 53.41291, "longitude": -8.24389}, - "eu-central-1": {"latitude": 50.1109221, "longitude": 8.6821267}, - "sa-east-1": {"latitude": -23.5505199, "longitude": -46.63330939999999}, - "ap-southeast-1": {"latitude": 1.352083, "longitude": 103.819836}, - "ca-central-1": {"latitude": 56.130366, "longitude": -106.346771}, - "ap-northeast-2": {"latitude": 37.566535, "longitude": 126.9779692}, - "us-west-2": {"latitude": 43.8041334, "longitude": -120.5542012}, - "us-gov-west-1": {"latitude": 37.09024, "longitude": -95.712891}, - "us-west-1": {"latitude": 38.8375215, "longitude": -120.8958242}, - "cn-north-1": {"latitude": 39.90419989999999, "longitude": 116.4073963}, - "ap-northeast-1": {"latitude": 35.7090259, "longitude": 139.7319925}, - "eu-west-2": {"latitude": 51.5073509, "longitude": -0.1277583}, - "eu-west-3": {"latitude": 48.856614, "longitude": 2.3522219}, +import "yunion.io/x/onecloud/pkg/cloudprovider" + +var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{ + "ap-south-1": {Latitude: 19.0759837, Longitude: 72.8776559, City: "", CountryCode: ""}, + "ap-northeast-3": {Latitude: 34.6937378, Longitude: 135.5021651, City: "", CountryCode: ""}, + "us-east-1": {Latitude: 37.4315734, Longitude: -78.6568942, City: "", CountryCode: ""}, + "us-east-2": {Latitude: 40.4172871, Longitude: -82.90712300000001, City: "", CountryCode: ""}, + "ap-southeast-2": {Latitude: -33.8688197, Longitude: 151.2092955, City: "", CountryCode: ""}, + "cn-northwest-1": {Latitude: 37.198731, Longitude: 106.1580937, City: "", CountryCode: "CN"}, + "eu-west-1": {Latitude: 53.41291, Longitude: -8.24389, City: "", CountryCode: ""}, + "eu-central-1": {Latitude: 50.1109221, Longitude: 8.6821267, City: "", CountryCode: ""}, + "sa-east-1": {Latitude: -23.5505199, Longitude: -46.63330939999999, City: "", CountryCode: ""}, + "ap-southeast-1": {Latitude: 1.352083, Longitude: 103.819836, City: "", CountryCode: ""}, + "ca-central-1": {Latitude: 56.130366, Longitude: -106.346771, City: "", CountryCode: ""}, + "ap-northeast-2": {Latitude: 37.566535, Longitude: 126.9779692, City: "", CountryCode: ""}, + "us-west-2": {Latitude: 43.8041334, Longitude: -120.5542012, City: "", CountryCode: ""}, + "us-gov-west-1": {Latitude: 37.09024, Longitude: -95.712891, City: "", CountryCode: ""}, + "us-west-1": {Latitude: 38.8375215, Longitude: -120.8958242, City: "", CountryCode: ""}, + "cn-north-1": {Latitude: 39.90419989999999, Longitude: 116.4073963, City: "", CountryCode: ""}, + "ap-northeast-1": {Latitude: 35.7090259, Longitude: 139.7319925, City: "", CountryCode: ""}, + "eu-west-2": {Latitude: 51.5073509, Longitude: -0.1277583, City: "", CountryCode: ""}, + "eu-west-3": {Latitude: 48.856614, Longitude: 2.3522219, City: "", CountryCode: ""}, } diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go index 37e131015f..d7ee3eac23 100644 --- a/pkg/util/aws/region.go +++ b/pkg/util/aws/region.go @@ -15,7 +15,7 @@ import ( "yunion.io/x/onecloud/pkg/compute/models" ) -var RegionLocations map[string]string = map[string]string{ +var RegionLocations = map[string]string{ "us-east-2": "美国东部(俄亥俄州)", "us-east-1": "美国东部(弗吉尼亚北部)", "us-west-1": "美国西部(加利福尼亚北部)", @@ -212,28 +212,11 @@ func (self *SRegion) GetMetadata() *jsonutils.JSONDict { return nil } -func (self *SRegion) GetLatitude() float32 { - if data, ok := LatitudeAndLongitude[self.RegionId]; !ok { - log.Debugf("Region %s not found in LatitudeAndLongitude", self.RegionId) - return 0.0 - } else if lat, ok := data["latitude"]; !ok { - log.Debugf("Region %s's latitude not found in LatitudeAndLongitude", self.RegionId) - return 0.0 - } else { - return lat - } -} - -func (self *SRegion) GetLongitude() float32 { - if data, ok := LatitudeAndLongitude[self.RegionId]; !ok { - log.Debugf("Region %s not found in LatitudeAndLongitude", self.RegionId) - return 0.0 - } else if lat, ok := data["longitude"]; !ok { - log.Debugf("Region %s's latitude not found in LatitudeAndLongitude", self.RegionId) - return 0.0 - } else { - return lat +func (self *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { + if info, ok := LatitudeAndLongitude[self.RegionId]; ok { + return info } + return cloudprovider.SGeographicInfo{} } func (self *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) { diff --git a/pkg/util/azure/region.go b/pkg/util/azure/region.go index 22bac5d9ae..640d1eaac4 100644 --- a/pkg/util/azure/region.go +++ b/pkg/util/azure/region.go @@ -117,12 +117,11 @@ func (self *SRegion) GetProvider() string { return CLOUD_PROVIDER_AZURE } -func (self *SRegion) GetLatitude() float32 { - return self.Latitude -} - -func (self *SRegion) GetLongitude() float32 { - return self.Longitude +func (self *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { + info := cloudprovider.SGeographicInfo{} + info.Latitude = self.Latitude + info.Longitude = self.Longitude + return info } func (self *SRegion) GetStatus() string { diff --git a/pkg/util/qcloud/latitud_and_longitude.go b/pkg/util/qcloud/latitud_and_longitude.go index bc5e9999e5..26257d325f 100644 --- a/pkg/util/qcloud/latitud_and_longitude.go +++ b/pkg/util/qcloud/latitud_and_longitude.go @@ -1,23 +1,25 @@ package qcloud -var LatitudeAndLongitude = map[string]map[string]float32{ - "ap-bangkok": {"latitude": 13.756330, "longitude": 100.501762}, // 腾讯云 亚太地区(曼谷) - "ap-beijing": {"latitude": 39.904202, "longitude": 116.407394}, // 腾讯云 华北地区(北京) - "ap-chengdu": {"latitude": 30.572815, "longitude": 104.066803}, // 腾讯云 西南地区(成都) - "ap-chongqing": {"latitude": 29.431585, "longitude": 106.912254}, // 腾讯云 西南地区(重庆) - "ap-guangzhou": {"latitude": 23.129110, "longitude": 113.264381}, // 腾讯云 华南地区(广州) - "ap-guangzhou-open": {"latitude": 23.126593, "longitude": 113.273415}, // 腾讯云 华南地区(广州Open) - "ap-hongkong": {"latitude": 22.396427, "longitude": 114.109497}, // 腾讯云 东南亚地区(香港) - "ap-mumbai": {"latitude": 19.075983, "longitude": 72.877655}, // 腾讯云 亚太地区(孟买) - "ap-seoul": {"latitude": 37.566536, "longitude": 126.977966}, // 腾讯云 东南亚地区(首尔) - "ap-shanghai": {"latitude": 31.230391, "longitude": 121.473701}, // 腾讯云 华东地区(上海) - "ap-shanghai-fsi": {"latitude": 31.311033, "longitude": 121.536217}, // 腾讯云 华东地区(上海金融) - "ap-shenzhen-fsi": {"latitude": 22.531544, "longitude": 114.025467}, // 腾讯云 华南地区(深圳金融) - "ap-singapore": {"latitude": 1.352083, "longitude": 103.819839}, // 腾讯云 东南亚地区(新加坡) - "ap-tokyo": {"latitude": 35.709026, "longitude": 139.731995}, // 腾讯云 亚太地区(东京) - "eu-frankfurt": {"latitude": 51.165691, "longitude": 10.451526}, // 腾讯云 欧洲地区(德国) - "eu-moscow": {"latitude": 55.755825, "longitude": 37.617298}, // 腾讯云 欧洲地区(莫斯科) - "na-ashburn": {"latitude": 37.431572, "longitude": -78.656891}, // 腾讯云 美国东部(弗吉尼亚) - "na-siliconvalley": {"latitude": 37.387474, "longitude": -122.057541}, // 腾讯云 美国西部(硅谷) - "na-toronto": {"latitude": 43.653225, "longitude": -79.383186}, // 腾讯云 北美地区(多伦多) +import "yunion.io/x/onecloud/pkg/cloudprovider" + +var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{ + "ap-bangkok": {Latitude: 13.756330, Longitude: 100.501762, City: "Bangkok", CountryCode: "TH"}, // 腾讯云 亚太地区(曼谷) + "ap-beijing": {Latitude: 39.904202, Longitude: 116.407394, City: "Beijing", CountryCode: "CN"}, // 腾讯云 华北地区(北京) + "ap-chengdu": {Latitude: 30.572815, Longitude: 104.066803, City: "Chengdu", CountryCode: "CN"}, // 腾讯云 西南地区(成都) + "ap-chongqing": {Latitude: 29.431585, Longitude: 106.912254, City: "Chongqing", CountryCode: "CN"}, // 腾讯云 西南地区(重庆) + "ap-guangzhou": {Latitude: 23.129110, Longitude: 113.264381, City: "Guangzhou", CountryCode: "CN"}, // 腾讯云 华南地区(广州) + "ap-guangzhou-open": {Latitude: 23.126593, Longitude: 113.273415, City: "Guangzhou", CountryCode: "CN"}, // 腾讯云 华南地区(广州Open) + "ap-hongkong": {Latitude: 22.396427, Longitude: 114.109497, City: "Hongkong", CountryCode: "HK"}, // 腾讯云 东南亚地区(香港) + "ap-mumbai": {Latitude: 19.075983, Longitude: 72.877655, City: "Mumbai", CountryCode: "IN"}, // 腾讯云 亚太地区(孟买) + "ap-seoul": {Latitude: 37.566536, Longitude: 126.977966, City: "Seoul", CountryCode: "KR"}, // 腾讯云 东南亚地区(首尔) + "ap-shanghai": {Latitude: 31.230391, Longitude: 121.473701, City: "Shanghai", CountryCode: "CN"}, // 腾讯云 华东地区(上海) + "ap-shanghai-fsi": {Latitude: 31.311033, Longitude: 121.536217, City: "Shanghai", CountryCode: "CN"}, // 腾讯云 华东地区(上海金融) + "ap-shenzhen-fsi": {Latitude: 22.531544, Longitude: 114.025467, City: "Shenzhen", CountryCode: "CN"}, // 腾讯云 华南地区(深圳金融) + "ap-singapore": {Latitude: 1.352083, Longitude: 103.819839, City: "Singapore", CountryCode: "SG"}, // 腾讯云 东南亚地区(新加坡) + "ap-tokyo": {Latitude: 35.709026, Longitude: 139.731995, City: "Tokyo", CountryCode: "JP"}, // 腾讯云 亚太地区(东京) + "eu-frankfurt": {Latitude: 51.165691, Longitude: 10.451526, City: "Frankfurt", CountryCode: "DE"}, // 腾讯云 欧洲地区(德国) + "eu-moscow": {Latitude: 55.755825, Longitude: 37.617298, City: "Moscow", CountryCode: "RU"}, // 腾讯云 欧洲地区(莫斯科) + "na-ashburn": {Latitude: 37.431572, Longitude: -78.656891, City: "Virgina", CountryCode: "US"}, // 腾讯云 美国东部(弗吉尼亚) + "na-siliconvalley": {Latitude: 37.387474, Longitude: -122.057541, City: "Siliconvalley", CountryCode: "US"}, // 腾讯云 美国西部(硅谷) + "na-toronto": {Latitude: 43.653225, Longitude: -79.383186, City: "Toronto", CountryCode: "CA"}, // 腾讯云 北美地区(多伦多) } diff --git a/pkg/util/qcloud/region.go b/pkg/util/qcloud/region.go index 75d5c94681..f9d57610df 100644 --- a/pkg/util/qcloud/region.go +++ b/pkg/util/qcloud/region.go @@ -394,18 +394,11 @@ func (self *SRegion) GetVpcs(vpcIds []string, offset int, limit int) ([]SVpc, in return vpcs, int(total), nil } -func (self *SRegion) GetLatitude() float32 { +func (self *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { if info, ok := LatitudeAndLongitude[self.Region]; ok { - return info["latitude"] + return info } - return 0.0 -} - -func (self *SRegion) GetLongitude() float32 { - if info, ok := LatitudeAndLongitude[self.Region]; ok { - return info["longitude"] - } - return 0.0 + return cloudprovider.SGeographicInfo{} } func (self *SRegion) GetMetadata() *jsonutils.JSONDict { From 4bd67d60ee5dfa78d1b697dfb793ecb17f8ca797 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 01:23:48 +0800 Subject: [PATCH 07/34] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9A=E7=94=B1?= =?UTF-8?q?=E4=BA=8Edatastore=20url=E4=BF=A1=E6=81=AF=E4=B8=8D=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E5=AF=BC=E8=87=B4=E9=80=9A=E8=BF=87vcenter=E5=88=9B?= =?UTF-8?q?=E5=BB=BA=E4=B8=BB=E6=9C=BA=E5=A4=B1=E8=B4=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/appsrv/handlerinfo.go | 2 +- pkg/compute/hostdrivers/esxi.go | 7 +++++++ pkg/compute/models/hosts.go | 2 +- pkg/compute/tasks/cloud_account_sync_task.go | 1 + pkg/util/esxi/storage.go | 18 +++++++++++++----- 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/appsrv/handlerinfo.go b/pkg/appsrv/handlerinfo.go index 6a7234ca65..048a4124cd 100644 --- a/pkg/appsrv/handlerinfo.go +++ b/pkg/appsrv/handlerinfo.go @@ -110,4 +110,4 @@ func (hi *SHandlerInfo) GetAppParams(params map[string]string, path []string) *S appParams.Params = params appParams.Path = path return &appParams -} \ No newline at end of file +} diff --git a/pkg/compute/hostdrivers/esxi.go b/pkg/compute/hostdrivers/esxi.go index d7d1158007..05bbb9e9ef 100644 --- a/pkg/compute/hostdrivers/esxi.go +++ b/pkg/compute/hostdrivers/esxi.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/httputils" + "github.com/golang-plus/errors" ) type SESXiHostDriver struct { @@ -62,6 +63,12 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo content.Format = cacheImage.GetFormat() storage := host.GetStorageByFilePath(storageCache.Path) + if storage == nil { + msg := fmt.Sprintf("fail to find storage for storageCache %s", storageCache.Path) + log.Errorf(msg) + return errors.New(msg) + } + accessInfo, err := host.GetCloudaccount().GetVCenterAccessInfo(storage.ExternalId) if err != nil { return err diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 9fa926df40..d9107bdbfe 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -35,7 +36,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/onecloud/pkg/appsrv" ) const ( diff --git a/pkg/compute/tasks/cloud_account_sync_task.go b/pkg/compute/tasks/cloud_account_sync_task.go index cf5dfe6591..0ffcf2ef83 100644 --- a/pkg/compute/tasks/cloud_account_sync_task.go +++ b/pkg/compute/tasks/cloud_account_sync_task.go @@ -91,6 +91,7 @@ func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Con err := skus.SyncSkusByProviderIds([]string{cloudprovider.Provider}) return nil, err }) + return } } self.SetStageComplete(ctx, nil) diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 34c786b937..58d61e6a71 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -1,12 +1,8 @@ package esxi import ( - "github.com/vmware/govmomi/vim25/mo" - "context" "fmt" - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/types" "io" "io/ioutil" "net/http" @@ -16,8 +12,14 @@ import ( "strconv" "strings" "time" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/types" + "github.com/vmware/govmomi/vim25/mo" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/vmdkutils" @@ -330,8 +332,14 @@ func (self *SDatastore) GetManagerId() string { return self.manager.providerId } +const dsPrefix = "ds://" + func (self *SDatastore) GetUrl() string { - return self.getDatastore().Info.GetDatastoreInfo().Url + url := self.getDatastore().Info.GetDatastoreInfo().Url + if strings.HasPrefix(url, dsPrefix) { + url = url[len(dsPrefix):] + } + return url } func (self *SDatastore) GetMountPoint() string { From dd2ce41fa227d1344e6419a6ce67a2a919e73d94 Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 12 Dec 2018 11:15:30 +0800 Subject: [PATCH 08/34] skus list bugfix --- pkg/compute/models/skus.go | 49 ++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index ea5ec58b24..8906c85f4a 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -250,25 +250,38 @@ func (self *SServerSkuManager) AllowGetPropertyInstanceSpecs(ctx context.Context func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { q := self.Query() - zone, err := query.GetString("zone") - if err == nil && len(zone) > 0 { - zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zone) - if err != nil { - if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), zone) - } - return nil, httperrors.NewGeneralError(err) - } - - q = q.Equals("zone_id", zoneObj.GetId()) + provider, _ := query.GetString("provider") + if inWhiteList(provider) { + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNull(q.Field("provider")), + sqlchemy.IsEmpty(q.Field("provider")), + )) } else { - return nil, httperrors.NewMissingParameterError("zone") + q = q.Equals("provider", provider) + } + + // 如果是查询私有云需要忽略zone参数 + zone := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) + if !inWhiteList(provider){ + if len(zone) > 0 { + zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zone) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), zone) + } + return nil, httperrors.NewGeneralError(err) + } + + q = q.Equals("zone_id", zoneObj.GetId()) + } else { + return nil, httperrors.NewMissingParameterError("zone") + } } skus := make([]SServerSku, 0) q = q.GroupBy(q.Field("cpu_core_count"), q.Field("memory_size_mb")) q = q.Asc(q.Field("cpu_core_count"), q.Field("memory_size_mb")) - err = q.All(&skus) + err := q.All(&skus) if err != nil { log.Errorf("%s", err) return nil, httperrors.NewBadRequestError("instance specs list query error") @@ -432,13 +445,16 @@ func (self *SServerSku) GetZoneExternalId() (string, error) { func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { provider := jsonutils.GetAnyString(query, []string{"provider"}) - // provider 参数为all时。表示查询所有instance type + queryDict := query.(*jsonutils.JSONDict) if provider == "" { q = q.Filter(sqlchemy.OR( sqlchemy.IsNull(q.Field("provider")), sqlchemy.IsEmpty(q.Field("provider")), )) - } else if provider != "all" { + } else if provider == "all" { + // provider 参数为all时。表示查询所有instance type. + queryDict.Remove("provider") + } else { q = q.Equals("provider", provider) } @@ -470,6 +486,9 @@ func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchem return nil, httperrors.NewGeneralError(err) } q = q.Equals("zone_id", zoneObj.GetId()) + } else { + queryDict.Remove("zone") + queryDict.Remove("zone_id") } return q, err From ad2de3384697067d40a68e06e7870c2a9e9612b3 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Wed, 12 Dec 2018 11:38:09 +0800 Subject: [PATCH 09/34] fix code --- Gopkg.lock | 4 ++-- pkg/compute/models/guests.go | 4 ++-- pkg/compute/models/snapshots.go | 9 ++++++--- .../x/pkg/util/osprofile/osprofile.go | 2 +- .../yunion.io/x/pkg/util/secrules/secrules.go | 19 +++++-------------- vendor/yunion.io/x/sqlchemy/update.go | 14 ++++++++------ 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 6752a09d06..38b16f9f6d 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1368,11 +1368,11 @@ [[projects]] branch = "master" - digest = "1:2fdf064ae928c1b311e67ec51e856ce655abc39e8eacf5e6b3f7bd0417fac0a6" + digest = "1:04973e1902449b00dd7f3a9ad0b2b892c637cc8bc2e0dc569a82026fe1c4a4b3" name = "yunion.io/x/sqlchemy" packages = ["."] pruneopts = "UT" - revision = "e22221d5efcc667e68b0fdeed19958e788c3ad63" + revision = "998e91b54b0b9441f21dc9c09036f875a02ef8c5" [[projects]] branch = "master" diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 1420bcc813..784bc461b9 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -689,7 +689,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m if err != nil { return nil, httperrors.NewInputParameterError("Invalid root image: %s", err) } - if len(diskConfig.ImageId) > 0 && diskConfig.DiskType != DISK_TYPE_SYS { + if len(diskConfig.SnapshotId) > 0 && diskConfig.DiskType != DISK_TYPE_SYS { return nil, httperrors.NewBadRequestError("Snapshot error: disk index 0 but disk type is %s", diskConfig.DiskType) } @@ -781,7 +781,7 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m return nil, httperrors.NewInputParameterError("parse disk description error %s", err) } if diskConfig.DiskType == DISK_TYPE_SYS { - return nil, httperrors.NewBadRequestError("Snapshot error: disk index %d > 0 but disk type is %s", idx, DISK_TYPE_SYS) + return nil, httperrors.NewBadRequestError("Snapshot error: disk index %d > 0 but disk type is %s", i+1, DISK_TYPE_SYS) } if len(diskConfig.Backend) == 0 { diskConfig.Backend = rootStorageType diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index 2a68fdc766..a4aa643515 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -251,12 +251,15 @@ func (self *SSnapshot) GetHost() *SHost { func (self *SSnapshotManager) AddRefCount(snapshotId string, count int) { iSnapshot, _ := self.FetchById(snapshotId) - snapshot := iSnapshot.(*SSnapshot) - if snapshot != nil { - self.TableSpec().Update(snapshot, func() error { + if iSnapshot != nil { + snapshot := iSnapshot.(*SSnapshot) + _, err := self.TableSpec().Update(snapshot, func() error { snapshot.RefCount += count return nil }) + if err != nil { + log.Errorf("Snapshot add refence count error: %s", err) + } } } diff --git a/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go b/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go index 7d7c92d87f..09cb355365 100644 --- a/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go +++ b/vendor/yunion.io/x/pkg/util/osprofile/osprofile.go @@ -106,7 +106,7 @@ func GetOSProfileFromImageProperties(imgProp map[string]string, hypervisor strin } var imgHypers []string imgHyperStr, ok := imgProp["hypervisor"] - if ok { + if ok && len(imgHyperStr) > 0 { imgHypers = strings.Split(imgHyperStr, ",") } else { imgHypers = []string{} diff --git a/vendor/yunion.io/x/pkg/util/secrules/secrules.go b/vendor/yunion.io/x/pkg/util/secrules/secrules.go index 037bb62d45..6437ff1ae9 100644 --- a/vendor/yunion.io/x/pkg/util/secrules/secrules.go +++ b/vendor/yunion.io/x/pkg/util/secrules/secrules.go @@ -127,21 +127,12 @@ func ParseSecurityRule(pattern string) (*SecurityRule, error) { return nil, ErrInvalidAction } } else if status == SEG_IP { - // NOTE regutils.MatchCIDR actually also matches IP address without prefix length if regutils.MatchCIDR(seg) { - if idx := strings.Index(seg, "/"); idx > -1 { - if _, ipnet, err := net.ParseCIDR(seg); err != nil { - return nil, ErrInvalidNet - } else { - rule.IPNet = ipnet - } - } else if ip := net.ParseIP(seg); ip != nil { - rule.IPNet = &net.IPNet{ - IP: ip, - Mask: net.CIDRMask(32, 32), - } - } else { - return nil, ErrInvalidIPAddr + _, rule.IPNet, _ = net.ParseCIDR(seg) + } else if regutils.MatchIPAddr(seg) { + rule.IPNet = &net.IPNet{ + IP: net.ParseIP(seg), + Mask: net.CIDRMask(32, 32), } } else { rule.IPNet = &net.IPNet{ diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index 4a7e253e9f..3a6512d2fb 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -85,12 +85,14 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, er k := c.Name() of := ofields[k] nf := fields[k] - if c.IsPrimary() && !c.IsZero(of) { // skip update primary key - primaries[k] = of - continue - } else if c.IsKeyIndex() && !c.IsZero(of) { - keyIndexes[k] = of - continue + if !gotypes.IsNil(of) { + if c.IsPrimary() && !c.IsZero(of) { // skip update primary key + primaries[k] = of + continue + } else if c.IsKeyIndex() && !c.IsZero(of) { + keyIndexes[k] = of + continue + } } nc, ok := c.(*SIntegerColumn) if ok && nc.IsAutoVersion { From cb0823eb70de6531bf3edfcfa8559dfd9da70681 Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 12 Dec 2018 15:41:22 +0800 Subject: [PATCH 10/34] name bugfix --- pkg/compute/models/skus.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index 8906c85f4a..88fc8df1d2 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -167,11 +167,6 @@ func (self *SServerSkuManager) ValidateCreateData(ctx context.Context, } // name 由服务器端生成 - _, err := data.GetString("name") - if err != nil { - data.Remove("name") - } - cpu, err := data.Int("cpu_core_count") if err != nil { return nil, httperrors.NewInputParameterError("cpu_core_count should not be empty") @@ -262,7 +257,7 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use // 如果是查询私有云需要忽略zone参数 zone := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) - if !inWhiteList(provider){ + if !inWhiteList(provider) { if len(zone) > 0 { zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zone) if err != nil { @@ -366,11 +361,6 @@ func (self *SServerSku) ValidateUpdateData( } // name 由服务器端生成 - _, err = data.GetString("name") - if err != nil { - data.Remove("name") - } - cpu, err := data.Int("cpu_core_count") if err != nil { cpu = int64(self.CpuCoreCount) From 779a588bcf9da4dc6482d89c525bb1938151afa7 Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 12 Dec 2018 15:43:13 +0800 Subject: [PATCH 11/34] make fmt --- pkg/compute/models/capabilities.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/compute/models/capabilities.go b/pkg/compute/models/capabilities.go index f0bdff951f..4462fe4104 100644 --- a/pkg/compute/models/capabilities.go +++ b/pkg/compute/models/capabilities.go @@ -11,9 +11,9 @@ import ( ) type SCapabilities struct { - Hypervisors []string `json:",allowempty"` - StorageTypes []string `json:",allowempty"` - GPUModels []string `json:",allowempty"` + Hypervisors []string `json:",allowempty"` + StorageTypes []string `json:",allowempty"` + GPUModels []string `json:",allowempty"` MinNicCount int MaxNicCount int MinDataDiskCount int From aceb20bf79e3782c7178d0fb381862231586dcf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 12 Dec 2018 16:30:05 +0800 Subject: [PATCH 12/34] =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E6=80=A7=E7=BB=91=E5=AE=9A=E5=A4=9A=E4=B8=AA=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guest_actions.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 19f0397e3e..8af1fb8dfa 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/onecloud/pkg/util/seclib2" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/util/billing" @@ -643,24 +644,30 @@ func (self *SGuest) PerformAddSecgroup(ctx context.Context, userCred mcclient.To return nil, httperrors.NewInputParameterError("Cannot assign security rules in status %s", self.Status) } - secgrpV := validators.NewModelIdOrNameValidator("secgrp", "secgroup", userCred.GetProjectId()) - if err := secgrpV.Validate(data.(*jsonutils.JSONDict)); err != nil { - return nil, err - } - maxCount := self.GetDriver().GetMaxSecurityGroupCount() if maxCount == 0 { return nil, httperrors.NewUnsupportOperationError("Cannot assign security group for this guest %s", self.Name) } + secgrps := []string{} + if err := data.Unmarshal(&secgrps, "secgrps"); err != nil { + return nil, httperrors.NewInputParameterError(err.Error()) + } + secgroups := self.GetSecgroups() - if len(secgroups) >= maxCount { + if len(secgroups)+len(secgrps) >= maxCount { return nil, httperrors.NewUnsupportOperationError("guest %s band to up to %d security groups", self.Name, maxCount) } - secgroup := secgrpV.Model.(*SSecurityGroup) - if _, err := GuestsecgroupManager.newGuestSecgroup(ctx, userCred, self, secgroup); err != nil { - return nil, httperrors.NewInputParameterError(err.Error()) + for _, _secgrp := range secgrps { + secgrp, err := SecurityGroupManager.FetchByIdOrName(userCred, _secgrp) + if err != nil { + return nil, httperrors.NewInputParameterError(err.Error()) + } + secgroup := secgrp.(*SSecurityGroup) + if _, err := GuestsecgroupManager.newGuestSecgroup(ctx, userCred, self, secgroup); err != nil { + return nil, httperrors.NewInputParameterError(err.Error()) + } } return nil, self.StartSyncTask(ctx, userCred, true, "") } From 1d013215b35ae60e696db3d98a8ec70c6b818ee2 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 08:33:50 +0000 Subject: [PATCH 13/34] gofmt --- pkg/compute/models/capabilities.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/compute/models/capabilities.go b/pkg/compute/models/capabilities.go index f0bdff951f..4462fe4104 100644 --- a/pkg/compute/models/capabilities.go +++ b/pkg/compute/models/capabilities.go @@ -11,9 +11,9 @@ import ( ) type SCapabilities struct { - Hypervisors []string `json:",allowempty"` - StorageTypes []string `json:",allowempty"` - GPUModels []string `json:",allowempty"` + Hypervisors []string `json:",allowempty"` + StorageTypes []string `json:",allowempty"` + GPUModels []string `json:",allowempty"` MinNicCount int MaxNicCount int MinDataDiskCount int From 4f107ee02c9cddfcc81e979ec016acdbb9282edd Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 08:10:14 +0000 Subject: [PATCH 14/34] =?UTF-8?q?lbagent:=20=E4=BB=85=E5=A4=84=E7=90=86man?= =?UTF-8?q?ager=5Fid=E4=B8=BA=E7=A9=BA=E7=9A=84API=E5=AF=B9=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 当前还没有manager_id字段，先留出 --- pkg/lbagent/models/reflect.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/lbagent/models/reflect.go b/pkg/lbagent/models/reflect.go index d412ebd2ce..34af04a0f9 100644 --- a/pkg/lbagent/models/reflect.go +++ b/pkg/lbagent/models/reflect.go @@ -88,7 +88,10 @@ func GetModels(opts *GetModelsOptions) error { listOptions := options.BaseListOptions{ Admin: options.Bool(true), Details: options.Bool(true), - Filter: []string{minUpdatedAtFilter(minUpdatedAt)}, + Filter: []string{ + minUpdatedAtFilter(minUpdatedAt), // order matters, filter.0 + "isempty(manager_id)", // len(manager_id) > 0 is for pubcloud objects + }, OrderBy: []string{"updated_at", "id"}, Order: "asc", Limit: options.Int(opts.BatchListSize), From c6dd2346764fd7a926c4f7aafdc8c1cb4013f9db Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 08:46:21 +0000 Subject: [PATCH 15/34] =?UTF-8?q?loadbalancerlisteners:=20=E6=94=B9?= =?UTF-8?q?=E6=AD=A3update=20enable=5Fhttp2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/loadbalancerlisteners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go index 4828b3ecea..93d849fef0 100644 --- a/pkg/compute/models/loadbalancerlisteners.go +++ b/pkg/compute/models/loadbalancerlisteners.go @@ -357,7 +357,7 @@ func (lblis *SLoadbalancerListener) ValidateUpdateData(ctx context.Context, user "certificate": certV, "tls_cipher_policy": tlsCipherPolicyV, - "enable_http2": validators.NewBoolValidator("enable_http2").Default(true), + "enable_http2": validators.NewBoolValidator("enable_http2"), } for _, v := range keyV { v.Optional(true) From 35384160ce8ee5691a881cc1670ad458b69e64b2 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 08:35:08 +0000 Subject: [PATCH 16/34] =?UTF-8?q?validators:=20=E6=B7=BB=E5=8A=A0ApplyMode?= =?UTF-8?q?lFilters()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudcommon/validators/misc.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 pkg/cloudcommon/validators/misc.go diff --git a/pkg/cloudcommon/validators/misc.go b/pkg/cloudcommon/validators/misc.go new file mode 100644 index 0000000000..0c36afbf5b --- /dev/null +++ b/pkg/cloudcommon/validators/misc.go @@ -0,0 +1,29 @@ +package validators + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/sqlchemy" +) + +type ModelFilterOptions struct { + Key string + ModelKeyword string + ProjectId string +} + +func ApplyModelFilters(q *sqlchemy.SQuery, data *jsonutils.JSONDict, opts []*ModelFilterOptions) (*sqlchemy.SQuery, error) { + var err error + for _, opt := range opts { + v := NewModelIdOrNameValidator( + opt.Key, + opt.ModelKeyword, + opt.ProjectId, + ) + v.Optional(true) + q, err = v.QueryFilter(q, data) + if err != nil { + return nil, err + } + } + return q, nil +} From 1a7283b8470f3d39c00df1b3bcf8d74f28ede890 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 08:35:35 +0000 Subject: [PATCH 17/34] =?UTF-8?q?loadbalancers:=20=E4=BD=BF=E7=94=A8ApplyM?= =?UTF-8?q?odelFilters()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../models/loadbalancerbackendgroups.go | 12 ++++---- pkg/compute/models/loadbalancerbackends.go | 22 ++++---------- .../models/loadbalancerlistenerrules.go | 21 ++++--------- pkg/compute/models/loadbalancerlisteners.go | 30 +++++-------------- pkg/compute/models/loadbalancers.go | 21 ++++--------- 5 files changed, 30 insertions(+), 76 deletions(-) diff --git a/pkg/compute/models/loadbalancerbackendgroups.go b/pkg/compute/models/loadbalancerbackendgroups.go index ff4917c158..e5f3c8a35b 100644 --- a/pkg/compute/models/loadbalancerbackendgroups.go +++ b/pkg/compute/models/loadbalancerbackendgroups.go @@ -52,13 +52,11 @@ func (man *SLoadbalancerBackendGroupManager) ListItemFilter(ctx context.Context, } userProjId := userCred.GetProjectId() data := query.(*jsonutils.JSONDict) - { - lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId) - lbV.Optional(true) - q, err = lbV.QueryFilter(q, data) - if err != nil { - return nil, err - } + q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{ + {Key: "loadbalancer", ModelKeyword: "loadbalancer", ProjectId: userProjId}, + }) + if err != nil { + return nil, err } return q, nil } diff --git a/pkg/compute/models/loadbalancerbackends.go b/pkg/compute/models/loadbalancerbackends.go index 4eb5dcba9d..3c03e56921 100644 --- a/pkg/compute/models/loadbalancerbackends.go +++ b/pkg/compute/models/loadbalancerbackends.go @@ -55,22 +55,12 @@ func (man *SLoadbalancerBackendManager) ListItemFilter(ctx context.Context, q *s } userProjId := userCred.GetProjectId() data := query.(*jsonutils.JSONDict) - { - backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId) - backendGroupV.Optional(true) - q, err = backendGroupV.QueryFilter(q, data) - if err != nil { - return nil, err - } - } - { - // NOTE extend this when new backend_type was added - backendV := validators.NewModelIdOrNameValidator("backend", "server", userProjId) - backendV.Optional(true) - q, err = backendV.QueryFilter(q, data) - if err != nil { - return nil, err - } + q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{ + {Key: "backend_group", ModelKeyword: "loadbalancerbackendgroup", ProjectId: userProjId}, + {Key: "backend", ModelKeyword: "server", ProjectId: userProjId}, // NOTE extend this when new backend_type was added + }) + if err != nil { + return nil, err } return q, nil } diff --git a/pkg/compute/models/loadbalancerlistenerrules.go b/pkg/compute/models/loadbalancerlistenerrules.go index 26f6540822..78cd70cf01 100644 --- a/pkg/compute/models/loadbalancerlistenerrules.go +++ b/pkg/compute/models/loadbalancerlistenerrules.go @@ -72,21 +72,12 @@ func (man *SLoadbalancerListenerRuleManager) ListItemFilter(ctx context.Context, } userProjId := userCred.GetProjectId() data := query.(*jsonutils.JSONDict) - { - listenerV := validators.NewModelIdOrNameValidator("listener", "loadbalancerlistener", userProjId) - listenerV.Optional(true) - q, err = listenerV.QueryFilter(q, data) - if err != nil { - return nil, err - } - } - { - backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId) - backendGroupV.Optional(true) - q, err = backendGroupV.QueryFilter(q, data) - if err != nil { - return nil, err - } + q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{ + {Key: "listener", ModelKeyword: "loadbalancerlistener", ProjectId: userProjId}, + {Key: "backend_group", ModelKeyword: "loadbalancerbackendgroup", ProjectId: userProjId}, + }) + if err != nil { + return nil, err } return q, nil } diff --git a/pkg/compute/models/loadbalancerlisteners.go b/pkg/compute/models/loadbalancerlisteners.go index 93d849fef0..df74938cba 100644 --- a/pkg/compute/models/loadbalancerlisteners.go +++ b/pkg/compute/models/loadbalancerlisteners.go @@ -143,29 +143,13 @@ func (man *SLoadbalancerListenerManager) ListItemFilter(ctx context.Context, q * } userProjId := userCred.GetProjectId() data := query.(*jsonutils.JSONDict) - { - lbV := validators.NewModelIdOrNameValidator("loadbalancer", "loadbalancer", userProjId) - lbV.Optional(true) - q, err = lbV.QueryFilter(q, data) - if err != nil { - return nil, err - } - } - { - backendGroupV := validators.NewModelIdOrNameValidator("backend_group", "loadbalancerbackendgroup", userProjId) - backendGroupV.Optional(true) - q, err = backendGroupV.QueryFilter(q, data) - if err != nil { - return nil, err - } - } - { - aclV := validators.NewModelIdOrNameValidator("acl", "loadbalanceracl", userProjId) - aclV.Optional(true) - q, err = aclV.QueryFilter(q, data) - if err != nil { - return nil, err - } + q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{ + {Key: "loadbalancer", ModelKeyword: "loadbalancer", ProjectId: userProjId}, + {Key: "backend_group", ModelKeyword: "loadbalancerbackendgroup", ProjectId: userProjId}, + {Key: "acl", ModelKeyword: "loadbalanceracl", ProjectId: userProjId}, + }) + if err != nil { + return nil, err } return q, nil } diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index c77e689677..a1b092dc0b 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -62,21 +62,12 @@ func (man *SLoadbalancerManager) ListItemFilter(ctx context.Context, q *sqlchemy } userProjId := userCred.GetProjectId() data := query.(*jsonutils.JSONDict) - { - networkV := validators.NewModelIdOrNameValidator("network", "network", userProjId) - networkV.Optional(true) - q, err = networkV.QueryFilter(q, data) - if err != nil { - return nil, err - } - } - { - zoneV := validators.NewModelIdOrNameValidator("zone", "zone", userProjId) - zoneV.Optional(true) - q, err = zoneV.QueryFilter(q, data) - if err != nil { - return nil, err - } + q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{ + {Key: "network", ModelKeyword: "network", ProjectId: userProjId}, + {Key: "zone", ModelKeyword: "zone", ProjectId: userProjId}, + }) + if err != nil { + return nil, err } return q, nil } From 09750de93436edac99890b252b7b4e1f0cd5b12b Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 16:51:59 +0800 Subject: [PATCH 18/34] =?UTF-8?q?=E6=80=BB=E6=98=AF=E5=85=81=E8=AE=B8?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E8=B0=83=E5=BA=A6=E6=A0=87=E7=AD=BE=EF=BC=8C?= =?UTF-8?q?=E6=97=A0=E8=AE=BA=E5=85=AC=E6=9C=89=E4=BA=91=E8=BF=98=E6=98=AF?= =?UTF-8?q?=E7=A7=81=E6=9C=89=E4=BA=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/capabilities.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkg/compute/models/capabilities.go b/pkg/compute/models/capabilities.go index 4462fe4104..d6fb559df4 100644 --- a/pkg/compute/models/capabilities.go +++ b/pkg/compute/models/capabilities.go @@ -132,11 +132,7 @@ func getNetworkCount(zone *SZone) int { } func isSchedPolicySupported(zone *SZone) bool { - if zone != nil { - return !zone.isManaged() - } else { - return true - } + return true } func getMinNicCount(zone *SZone) int { From 7a3ddc7caf8afe83657797d21995ed20b5e42501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 12 Dec 2018 17:00:02 +0800 Subject: [PATCH 19/34] =?UTF-8?q?=E9=81=BF=E5=85=8D=E5=90=8C=E6=AD=A5cache?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E5=AE=89=E5=85=A8=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/secgroupcache.go | 17 +++++++++++++++++ pkg/compute/models/secgroups.go | 16 ++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index 41d785b68c..7f93963a30 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -123,6 +123,23 @@ func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, return &secgroupCache } +func (manager *SSecurityGroupCacheManager) CheckExist(ctx context.Context, userCred mcclient.TokenCredential, externalId, vpcId, regionId string, providerId string) (*SSecurityGroup, bool) { + secgroupCaches := []SSecurityGroupCache{} + query := manager.Query() + cond := sqlchemy.AND(sqlchemy.Equals(query.Field("external_id"), externalId), sqlchemy.Equals(query.Field("vpc_id"), vpcId), sqlchemy.Equals(query.Field("cloudregion_id"), regionId), sqlchemy.Equals(query.Field("manager_id"), providerId)) + query = query.Filter(cond) + + if err := query.All(&secgroupCaches); err != nil { + return nil, false + } + for _, secgroupCache := range secgroupCaches { + if secgroup, err := SecurityGroupManager.FetchById(secgroupCache.SecgroupId); err == nil { + return secgroup.(*SSecurityGroup), true + } + } + return nil, false +} + func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string) *SSecurityGroupCache { lockman.LockClass(ctx, manager, userCred.GetProjectId()) defer lockman.ReleaseClass(ctx, manager, userCred.GetProjectId()) diff --git a/pkg/compute/models/secgroups.go b/pkg/compute/models/secgroups.go index aa7ef6be88..e17f5502e7 100644 --- a/pkg/compute/models/secgroups.go +++ b/pkg/compute/models/secgroups.go @@ -218,7 +218,12 @@ func (self *SSecurityGroup) SyncWithCloudSecurityGroup(userCred mcclient.TokenCr return nil } -func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc) (*SSecurityGroup, error) { +func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCredential, extSec cloudprovider.ICloudSecurityGroup, vpc *SVpc) (*SSecurityGroup, bool, error) { + if secgroup, exist := SecurityGroupCacheManager.CheckExist(context.Background(), userCred, extSec.GetGlobalId(), extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); exist { + //避免重复同步 + return secgroup, true, nil + } + secgroup := SSecurityGroup{} secgroup.SetModelManager(manager) secgroup.Name = extSec.GetName() @@ -227,7 +232,7 @@ func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCre secgroup.ProjectId = userCred.GetProjectId() if err := manager.TableSpec().Insert(&secgroup); err != nil { - return nil, err + return nil, false, err } if secgroupcache := SecurityGroupCacheManager.Register(context.Background(), userCred, secgroup.Id, extSec.GetVpcId(), vpc.CloudregionId, vpc.ManagerId); secgroupcache != nil { @@ -236,7 +241,7 @@ func (manager *SSecurityGroupManager) newFromCloudVpc(userCred mcclient.TokenCre } } - return &secgroup, nil + return &secgroup, false, nil } func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCred mcclient.TokenCredential, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { @@ -280,11 +285,14 @@ func (manager *SSecurityGroupManager) SyncSecgroups(ctx context.Context, userCre syncResult.AddError(err) continue } - new, err := manager.newFromCloudVpc(userCred, added[i], vpc) + new, exist, err := manager.newFromCloudVpc(userCred, added[i], vpc) if err != nil { syncResult.AddError(err) continue } + if exist { + continue + } localSecgroups = append(localSecgroups, *new) remoteSecgroups = append(remoteSecgroups, added[i]) SecurityGroupRuleManager.SyncRules(ctx, userCred, new, rules) From b64c3b333371987aea51e25d6bc0104194e10ccb Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 12 Dec 2018 17:46:56 +0800 Subject: [PATCH 20/34] region: specs panic when host sysinfo is nil --- pkg/compute/models/hosts.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index a1df23d231..4813953757 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -733,16 +733,21 @@ func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { } } spec.Set("nic_count", jsonutils.NewInt(nicCount)) - manufacture, err := self.SysInfo.Get("manufacture") - if err != nil { - manufacture = jsonutils.NewString("Unknown") + + var manufacture string + var model string + if self.SysInfo != nil { + manufacture, _ = self.SysInfo.GetString("manufacture") + model, _ = self.SysInfo.GetString("model") } - spec.Set("manufacture", manufacture) - model, err := self.SysInfo.Get("model") - if err != nil { - model = jsonutils.NewString("Unknown") + if manufacture == "" { + manufacture = "Unknown" } - spec.Set("model", model) + if model == "" { + model = "Unknown" + } + spec.Set("manufacture", jsonutils.NewString(manufacture)) + spec.Set("model", jsonutils.NewString(model)) return spec } From e0d7cbb431f443f050c2bc8cf5cca37d83b79dc7 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 19:23:03 +0800 Subject: [PATCH 21/34] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9Ahost-undo-recy?= =?UTF-8?q?cle=20fail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/hoststorages.go | 6 +++--- pkg/compute/models/host_recycle.go | 17 +++++++++++------ pkg/compute/models/hosts.go | 20 ++++++++++++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/cmd/climc/shell/hoststorages.go b/cmd/climc/shell/hoststorages.go index 88d25a21a9..a524a911cd 100644 --- a/cmd/climc/shell/hoststorages.go +++ b/cmd/climc/shell/hoststorages.go @@ -25,9 +25,9 @@ func init() { } var result *modules.ListResult var err error - if len(args.Storage) > 0 { - params.Add(jsonutils.NewString(args.Storage), "storage") - } + // if len(args.Storage) > 0 { + // params.Add(jsonutils.NewString(args.Storage), "storage") + // } if len(args.Host) > 0 { result, err = modules.Hoststorages.ListDescendent(s, args.Host, params) } else if len(args.Storage) > 0 { diff --git a/pkg/compute/models/host_recycle.go b/pkg/compute/models/host_recycle.go index a3d9076c42..f50af5820c 100644 --- a/pkg/compute/models/host_recycle.go +++ b/pkg/compute/models/host_recycle.go @@ -11,6 +11,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/tristate" + "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -351,7 +352,10 @@ func doUndoPrepaidRecycle(ctx context.Context, userCred mcclient.TokenCredential q := HostManager.Query() q = q.Equals("external_id", host.ExternalId) q = q.Equals("host_type", host.HostType) - q = q.IsNullOrEmpty("resource_type") + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNullOrEmpty(q.Field("resource_type")), + sqlchemy.Equals(q.Field("resource_type"), HostResourceTypeShared), + )) oHostCnt := q.Count() @@ -428,13 +432,14 @@ func doUndoPrepaidRecycle(ctx context.Context, userCred mcclient.TokenCredential return err } - oStorageObj, err := StorageManager.FetchByExternalId(istorage.GetGlobalId()) - if err != nil { - log.Errorf("StorageManager.FetchByExternalId fail %s", err) - return err + oHostStorage := oHost.GetHoststorageByExternalId(istorage.GetGlobalId()) + if oHostStorage == nil { + msg := fmt.Sprintf("oHost.GetHoststorageByExternalId not found %s", istorage.GetGlobalId()) + log.Errorf(msg) + return errors.New(msg) } - oStorage := oStorageObj.(*SStorage) + oStorage := oHostStorage.GetStorage() if storage.StorageType == STORAGE_LOCAL { _, err = disk.GetModelManager().TableSpec().Update(disk, func() error { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index a1df23d231..11844889d0 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -502,6 +502,26 @@ func (self *SHost) GetHoststorageOfId(storageId string) *SHoststorage { return &hoststorage } +func (self *SHost) GetHoststorageByExternalId(extId string) *SHoststorage { + hoststorage := SHoststorage{} + hoststorage.SetModelManager(HoststorageManager) + + hoststorages := HoststorageManager.Query().SubQuery() + storages := StorageManager.Query().SubQuery() + q := hoststorages.Query() + q = q.Join(storages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id"))) + q = q.Filter(sqlchemy.Equals(hoststorages.Field("host_id"), self.Id)) + q = q.Filter(sqlchemy.Equals(storages.Field("external_id"), extId)) + + err := q.First(&hoststorage) + if err != nil { + log.Errorf("GetHoststorageByExternalId fail %s", err) + return nil + } + + return &hoststorage +} + func (self *SHost) GetStorageByFilePath(path string) *SStorage { hoststorages := self.GetHoststorages() if hoststorages == nil { From 75f04cae51fcd468020cb3a8d3dbd43f421a1c60 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 19:55:35 +0800 Subject: [PATCH 22/34] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=EF=BC=9Aprepaid=20recy?= =?UTF-8?q?cle=20host=E7=9A=84storage=20driver=E8=AE=BE=E7=BD=AE=E4=B8=BAL?= =?UTF-8?q?inux?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/host_recycle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/host_recycle.go b/pkg/compute/models/host_recycle.go index f50af5820c..4b2284df4a 100644 --- a/pkg/compute/models/host_recycle.go +++ b/pkg/compute/models/host_recycle.go @@ -108,7 +108,7 @@ func (self *SGuest) doPrepaidRecycleNoLock(ctx context.Context, userCred mcclien info.Size = int64(disk.DiskSize) info.Index = int64(i) info.Slot = i - info.Driver = storage.StorageType + info.Driver = baremetal.DISK_DRIVER_LINUX info.Rotate = (storage.MediumType != DISK_TYPE_SSD) storageInfo = append(storageInfo, info) From 2c5ed4b3607a96109a4e6f288ba1dddfd45b04a8 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 11 Dec 2018 18:23:09 +0800 Subject: [PATCH 23/34] scheduler: make schedtag prefer and avoid strategy work --- pkg/scheduler/algorithm/plugin/plugin.go | 12 + .../predicates/aggregate_predicate.go | 56 ++-- .../predicates/baremetal/network_predicate.go | 10 +- .../predicates/guest/group_predicate.go | 50 ++- .../predicates/guest/network_predicate.go | 19 +- .../priorities/guest/avoid_same_host.go | 2 +- .../algorithm/priorities/guest/capacity.go | 2 +- .../algorithm/priorities/guest/creating.go | 4 +- .../algorithm/priorities/guest/lowload.go | 7 +- .../algorithm/priorities/priorities.go | 49 ++- pkg/scheduler/algorithmprovider/defaults.go | 1 - pkg/scheduler/core/context.go | 95 +++--- pkg/scheduler/core/generic_scheduler.go | 63 ++-- pkg/scheduler/core/score/score.go | 294 ++++++++++++++++++ pkg/scheduler/core/score/score_test.go | 156 ++++++++++ pkg/scheduler/core/types.go | 17 +- pkg/util/hashcache/doc.go | 1 + 17 files changed, 642 insertions(+), 196 deletions(-) create mode 100644 pkg/scheduler/algorithm/plugin/plugin.go create mode 100644 pkg/scheduler/core/score/score.go create mode 100644 pkg/scheduler/core/score/score_test.go create mode 100644 pkg/util/hashcache/doc.go diff --git a/pkg/scheduler/algorithm/plugin/plugin.go b/pkg/scheduler/algorithm/plugin/plugin.go new file mode 100644 index 0000000000..7eba61ab19 --- /dev/null +++ b/pkg/scheduler/algorithm/plugin/plugin.go @@ -0,0 +1,12 @@ +package plugin + +import ( + "yunion.io/x/onecloud/pkg/scheduler/core" +) + +type BasePlugin struct{} + +// Customize priority +func (p BasePlugin) OnPriorityEnd(u *core.Unit, c core.Candidater) {} + +func (p BasePlugin) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {} diff --git a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go index 3b869f94bd..6563aaf7b0 100644 --- a/pkg/scheduler/algorithm/predicates/aggregate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/aggregate_predicate.go @@ -6,8 +6,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" + "yunion.io/x/onecloud/pkg/scheduler/core/score" "yunion.io/x/onecloud/pkg/scheduler/db/models" "yunion.io/x/onecloud/pkg/util/conditionparser" ) @@ -23,6 +25,7 @@ import ( // the host is available. type AggregatePredicate struct { BasePredicate + plugin.BasePlugin AggregateHosts hostsAggregatesMap RequireAggregates []api.Aggregate ExcludeAggregates []api.Aggregate @@ -193,33 +196,6 @@ func getHostAggregateCount(inAggs []api.Aggregate, hAggs []*models.Aggregate, st return } -func (p *AggregatePredicate) OnSelect(u *core.Unit, c core.Candidater) bool { - hostAggs, ok := p.AggregateHosts[c.IndexKey()] - if !ok { - return true - } - - avoidCountMap := getHostAggregateCount(p.AvoidAggregates, hostAggs, api.AggregateStrategyAvoid) - preferCountMap := getHostAggregateCount(p.PreferAggregates, hostAggs, api.AggregateStrategyPrefer) - - setScore := func(aggCountMap map[string]int, postiveScore bool) { - stepScore := core.PriorityStep - if !postiveScore { - stepScore = -stepScore - } - for n, count := range aggCountMap { - u.IncreaseScore(c.IndexKey(), n, count*stepScore) - } - } - - setScore(avoidCountMap, false) - setScore(preferCountMap, true) - - return true -} - -func (p *AggregatePredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) {} - func (p *AggregatePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(p, u, c) @@ -281,3 +257,29 @@ func (p *AggregatePredicate) exec(h *PredicateHelper) string { return "" } + +func (p *AggregatePredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) { + hostAggs, ok := p.AggregateHosts[c.IndexKey()] + if !ok { + return + } + + avoidCountMap := getHostAggregateCount(p.AvoidAggregates, hostAggs, api.AggregateStrategyAvoid) + preferCountMap := getHostAggregateCount(p.PreferAggregates, hostAggs, api.AggregateStrategyPrefer) + + setScore := func(aggCountMap map[string]int, postiveScore bool) { + stepScore := core.PriorityStep + if !postiveScore { + stepScore = -stepScore + } + for n, count := range aggCountMap { + u.SetFrontScore( + c.IndexKey(), + score.NewScore(score.TScore(count*stepScore), n), + ) + } + } + + setScore(preferCountMap, true) + setScore(avoidCountMap, false) +} diff --git a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go index 45f73219e7..973244cc0b 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/network_predicate.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" @@ -14,6 +15,7 @@ import ( type NetworkPredicate struct { BasePredicate + plugin.BasePlugin SelectedNetworks sync.Map } @@ -141,11 +143,3 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor return h.GetResult() } - -func (p *NetworkPredicate) OnSelect(u *core.Unit, c core.Candidater) bool { - u.SetFiltedData(c.IndexKey(), "networks", &p.SelectedNetworks) - return true -} - -func (p *NetworkPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { -} diff --git a/pkg/scheduler/algorithm/predicates/guest/group_predicate.go b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go index 3233bc1f77..9c9610c73c 100644 --- a/pkg/scheduler/algorithm/predicates/guest/group_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/group_predicate.go @@ -3,14 +3,17 @@ package guest import ( "fmt" + "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" "yunion.io/x/onecloud/pkg/scheduler/core" + "yunion.io/x/onecloud/pkg/scheduler/core/score" ) // GroupPredicate filter the packet based on the label information, // the same group of guests should avoid schedule on same host. type GroupPredicate struct { predicates.BasePredicate + plugin.BasePlugin ExcludeGroups []string RequireGroups []string @@ -47,33 +50,6 @@ func (p *GroupPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, e return true, nil } -func (p *GroupPredicate) OnSelect(u *core.Unit, c core.Candidater) bool { - if len(p.ExcludeGroups) > 0 { - return false - } - - if len(p.RequireGroups) > 0 { - // TODO: what? - } - - if len(p.AvoidGroups) > 0 { - u.IncreaseScore(c.IndexKey(), - p.Name()+":avoid", -core.PriorityStep*len(p.AvoidGroups), - ) - } - - if len(p.PreferGroups) > 0 { - u.IncreaseScore(c.IndexKey(), - p.Name()+":prefer", core.PriorityStep*len(p.PreferGroups), - ) - } - - return true -} - -func (p *GroupPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { -} - func (p *GroupPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := predicates.NewPredicateHelper(p, u, c) @@ -101,3 +77,23 @@ func (p *GroupPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core. return h.GetResult() } + +func (p *GroupPredicate) OnPriorityEnd(u *core.Unit, c core.Candidater) { + if len(p.AvoidGroups) > 0 { + u.SetFrontScore( + c.IndexKey(), + score.NewScore( + score.TScore(-core.PriorityStep*len(p.AvoidGroups)), + p.Name()+":avoid", + )) + } + + if len(p.PreferGroups) > 0 { + u.SetFrontScore( + c.IndexKey(), + score.NewScore( + score.TScore(core.PriorityStep*len(p.PreferGroups)), + p.Name()+":prefer", + )) + } +} diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index be04fba623..ab3b888cc1 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -5,12 +5,14 @@ import ( "strings" "sync" + "yunion.io/x/pkg/util/sets" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" networks "yunion.io/x/onecloud/pkg/scheduler/db/models" - "yunion.io/x/pkg/util/sets" - "yunion.io/x/pkg/utils" ) // NetworkPredicate will filter the current network information with @@ -18,6 +20,7 @@ import ( // randomly match the available network resources. type NetworkPredicate struct { predicates.BasePredicate + plugin.BasePlugin SelectedNetworks sync.Map } @@ -105,10 +108,6 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor p.SelectedNetworks.Store(n.ID, counter.GetCount()) counters.Add(counter) found = true - - if counters.GetCount() >= d.Count { - break - } } else { fullErrMsgs = append(fullErrMsgs, fmt.Sprintf("%s: %s", n.ID, strings.Join(errMsgs, ",")), @@ -202,11 +201,3 @@ func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor return h.GetResult() } - -func (p *NetworkPredicate) OnSelect(u *core.Unit, c core.Candidater) bool { - u.SetFiltedData(c.IndexKey(), "networks", &p.SelectedNetworks) - return true -} - -func (p *NetworkPredicate) OnSelectEnd(u *core.Unit, c core.Candidater, count int64) { -} diff --git a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go index eb784a4213..46698dd13d 100644 --- a/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go +++ b/pkg/scheduler/algorithm/priorities/guest/avoid_same_host.go @@ -27,7 +27,7 @@ func (p *AvoidSameHostPriority) Map(u *core.Unit, c core.Candidater) (core.HostP ownerTenantID := u.SchedData().OwnerTenantID if count, ok := hc.Tenants[ownerTenantID]; ok && count > 0 { - h.SetScore(-50 * int(count)) + h.SetFrontRawScore(-1 * int(count)) } return h.GetResult() diff --git a/pkg/scheduler/algorithm/priorities/guest/capacity.go b/pkg/scheduler/algorithm/priorities/guest/capacity.go index 6b0201ab71..cb66c496a6 100644 --- a/pkg/scheduler/algorithm/priorities/guest/capacity.go +++ b/pkg/scheduler/algorithm/priorities/guest/capacity.go @@ -21,7 +21,7 @@ func (p *CapacityPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriori h := priorities.NewPriorityHelper(p, u, c) capacity := u.GetCapacity(c.IndexKey()) - h.SetScore(50 * int(capacity)) + h.SetRawScore(int(capacity)) return h.GetResult() } diff --git a/pkg/scheduler/algorithm/priorities/guest/creating.go b/pkg/scheduler/algorithm/priorities/guest/creating.go index 7e3dc12c8e..9d6fe061ba 100644 --- a/pkg/scheduler/algorithm/priorities/guest/creating.go +++ b/pkg/scheduler/algorithm/priorities/guest/creating.go @@ -27,8 +27,8 @@ func (p *CreatingPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriori } if hc.CreatingGuestCount > 0 { - score := -int(hc.CreatingGuestCount) * 20 - h.SetScore(score) + score := -int(hc.CreatingGuestCount) + h.SetFrontScore(score) } return h.GetResult() diff --git a/pkg/scheduler/algorithm/priorities/guest/lowload.go b/pkg/scheduler/algorithm/priorities/guest/lowload.go index cf74609fec..79d56f6c84 100644 --- a/pkg/scheduler/algorithm/priorities/guest/lowload.go +++ b/pkg/scheduler/algorithm/priorities/guest/lowload.go @@ -3,6 +3,7 @@ package guest import ( "yunion.io/x/onecloud/pkg/scheduler/algorithm/priorities" "yunion.io/x/onecloud/pkg/scheduler/core" + "yunion.io/x/onecloud/pkg/scheduler/core/score" ) type LowLoadPriority struct { @@ -28,8 +29,12 @@ func (p *LowLoadPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriorit cpuCommitRate := float64(hc.RunningCPUCount) / float64(hc.TotalCPUCount) memCommitRate := float64(hc.RunningMemSize) / float64(hc.TotalMemSize) if cpuCommitRate < 0.5 && memCommitRate < 0.5 { - score := 20 * (1 - cpuCommitRate - memCommitRate) + score := 10 * (1 - cpuCommitRate - memCommitRate) h.SetScore(int(score)) } return h.GetResult() } + +func (p *LowLoadPriority) ScoreIntervals() score.Intervals { + return score.NewIntervals(0, 1, 5) +} diff --git a/pkg/scheduler/algorithm/priorities/priorities.go b/pkg/scheduler/algorithm/priorities/priorities.go index 8b3560a0de..cc18857c11 100644 --- a/pkg/scheduler/algorithm/priorities/priorities.go +++ b/pkg/scheduler/algorithm/priorities/priorities.go @@ -1,23 +1,18 @@ package priorities import ( - "math" - "yunion.io/x/onecloud/pkg/scheduler/algorithm" "yunion.io/x/onecloud/pkg/scheduler/cache/candidate" "yunion.io/x/onecloud/pkg/scheduler/core" + "yunion.io/x/onecloud/pkg/scheduler/core/score" ) -func aggPriority(x float64) float64 { - return math.Log(x + math.Sqrt(x*x+1)) -} - // PriorityHelper is a struct that as a base interface for all priorities. type PriorityHelper struct { priority core.Priority unit *core.Unit Candidate core.Candidater - score int + score score.SScore err error } @@ -29,9 +24,38 @@ func NewPriorityHelper(p core.Priority, u *core.Unit, c core.Candidater) *Priori } } -func (h *PriorityHelper) SetScore(score int) { - h.score = score - h.unit.SetScore(h.Candidate.IndexKey(), h.priority.Name(), score) +func (h *PriorityHelper) setIntervalScore(val int) score.SScore { + h.score = score.NewScore( + h.priority.ScoreIntervals().ToScore(int64(val)), + h.priority.Name()) + return h.score +} + +func (h *PriorityHelper) setRawScore(val int) score.SScore { + h.score = score.NewScore( + score.TScore(val), + h.priority.Name()) + return h.score +} + +func (h *PriorityHelper) SetScore(val int) { + h.setIntervalScore(val) + h.unit.SetScore(h.Candidate.IndexKey(), h.score) +} + +func (h *PriorityHelper) SetFrontScore(val int) { + h.setIntervalScore(val) + h.unit.SetFrontScore(h.Candidate.IndexKey(), h.score) +} + +func (h *PriorityHelper) SetRawScore(val int) { + h.setRawScore(val) + h.unit.SetScore(h.Candidate.IndexKey(), h.score) +} + +func (h *PriorityHelper) SetFrontRawScore(val int) { + h.setRawScore(val) + h.unit.SetFrontScore(h.Candidate.IndexKey(), h.score) } func (h *PriorityHelper) SetError(err error) { @@ -41,7 +65,6 @@ func (h *PriorityHelper) SetError(err error) { func (h *PriorityHelper) GetResult() (core.HostPriority, error) { return core.HostPriority{ Host: h.Candidate.IndexKey(), - Score: h.score, Candidate: h.Candidate, }, h.err } @@ -71,3 +94,7 @@ func (b *BasePriority) Name() string { func (b *BasePriority) HostCandidate(c core.Candidater) (*candidate.HostDesc, error) { return algorithm.ToHostCandidate(c) } + +func (b *BasePriority) ScoreIntervals() score.Intervals { + return score.NewIntervals(0, 1, 2) +} diff --git a/pkg/scheduler/algorithmprovider/defaults.go b/pkg/scheduler/algorithmprovider/defaults.go index 6da42fc24b..2eec8ec4ed 100644 --- a/pkg/scheduler/algorithmprovider/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults.go @@ -31,7 +31,6 @@ func defaultPredicates() sets.String { func defaultPriorities() sets.String { return sets.NewString( - factory.RegisterPriority("guest-avoid-same-cluster", &priorityguest.AvoidSameClusterPriority{}, 1), factory.RegisterPriority("guest-avoid-same-host", &priorityguest.AvoidSameHostPriority{}, 1), factory.RegisterPriority("guest-lowload", &priorityguest.LowLoadPriority{}, 1), factory.RegisterPriority("guest-creating", &priorityguest.CreatingPriority{}, 1), diff --git a/pkg/scheduler/core/context.go b/pkg/scheduler/core/context.go index de1044dbdc..a0f4f30ff9 100644 --- a/pkg/scheduler/core/context.go +++ b/pkg/scheduler/core/context.go @@ -9,17 +9,15 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/scheduler/api" + "yunion.io/x/onecloud/pkg/scheduler/core/score" ) const ( - EmptyScore int = 0x7FFFFFFFFFFFFFFF - BaseScore int = 10000 EmptyCapacity int64 = -1 MaxCapacity int64 = 0x7FFFFFFFFFFFFFFF ) var ( - EmptyScores = make(map[string]int) EmptyCapacities = make(map[string]Counter) ) @@ -186,8 +184,19 @@ type Capacity struct { } type Score struct { - Values map[string]int - Sum int + *score.ScoreBucket +} + +func newScore() *Score { + return &Score{ + ScoreBucket: score.NewScoreBuckets(), + } +} + +func newZeroScore() Score { + s := newScore() + s.Append(score.NewZeroScore()) + return *s } type SchedContextDataItem struct { @@ -484,7 +493,11 @@ func validateCapacityInput(c Counter) bool { return false } -func (u *Unit) SetScore(id, name string, score int) error { +type ScoreValue struct { + value score.TScore +} + +func (u *Unit) setScore(id string, val score.SScore, tofront bool) { u.scoreLock.Lock() defer u.scoreLock.Unlock() @@ -494,75 +507,45 @@ func (u *Unit) SetScore(id, name string, score int) error { ) if scoreObj, ok = u.ScoreMap[id]; !ok { - scoreObj = Score{Values: make(map[string]int), Sum: EmptyScore} + scoreObj = *newScore() u.ScoreMap[id] = scoreObj } - scoreObj.Values[name] = score - scoreObj.Sum = EmptyScore - - log.V(10).Infof("%q SetScore: %q -> %d", name, id, score) - return nil -} - -func (u *Unit) IncreaseScore(id string, name string, increase int) error { - - u.scoreLock.Lock() - defer u.scoreLock.Unlock() - - var ( - scoreObj Score - ok bool - ) - - score := int(0) - if scoreObj, ok = u.ScoreMap[id]; !ok { - scoreObj = Score{Values: make(map[string]int), Sum: EmptyScore} - u.ScoreMap[id] = scoreObj - score = increase + if tofront { + scoreObj.AddToFirst(val) } else { - if value, ok := scoreObj.Values[name]; ok { - score = value + increase - } else { - score = increase - } + scoreObj.SetScore(val) } - scoreObj.Values[name] = score - scoreObj.Sum = EmptyScore - - log.V(10).Infof("%q IncreaseScore: %q -> %d", name, id, score) - return nil + log.V(10).Infof("SetScore: %q -> %s", id, val.String()) } -func (u *Unit) GetScore(id string) int { +func (u *Unit) SetScore(id string, val score.SScore) { + u.setScore(id, val, false) +} + +func (u *Unit) SetFrontScore(id string, val score.SScore) { + u.setScore(id, val, true) +} + +func (u *Unit) GetScore(id string) Score { var ( scoreObj Score ok bool ) if scoreObj, ok = u.ScoreMap[id]; !ok { - return BaseScore + return *newScore() } - - if scoreObj.Sum == EmptyScore { - sum := int(0) - for _, value := range scoreObj.Values { - sum += value - } - - scoreObj.Sum = sum - } - - return scoreObj.Sum + BaseScore + return scoreObj } -func (u *Unit) GetScores(id string) map[string]int { - if scores, ok := u.ScoreMap[id]; ok { - return scores.Values +func (u *Unit) GetScoreDetails(id string) string { + if score, ok := u.ScoreMap[id]; ok { + return score.String() } - return EmptyScores + return "EmptyScore" } func (u *Unit) SetFiltedData(id string, name string, data interface{}) error { u.scoreLock.Lock() diff --git a/pkg/scheduler/core/generic_scheduler.go b/pkg/scheduler/core/generic_scheduler.go index bf02d7145f..f5533ef3e4 100644 --- a/pkg/scheduler/core/generic_scheduler.go +++ b/pkg/scheduler/core/generic_scheduler.go @@ -169,13 +169,13 @@ func newSchedResultByCtx(u *Unit, count int64, c Candidater) *SchedResultItem { Count: count, Capacity: u.GetCapacity(id), Name: fmt.Sprintf("%v", c.Get("Name")), - Score: u.GetScore(id), + Score: u.GetScore(id).DigitString(), Data: u.GetFiltedData(id, count), } if showDetails { r.CapacityDetails = GetCapacities(u, id) - r.ScoreDetails = u.GetScores(id) + r.ScoreDetails = u.GetScoreDetails(id) } return r } @@ -229,10 +229,10 @@ type SchedResultItem struct { Count int64 `json:"count"` Data map[string]interface{} `json:"data"` Capacity int64 `json:"capacity"` - Score int `json:"score"` + Score string `json:"score"` CapacityDetails map[string]int64 `json:"capacity_details"` - ScoreDetails map[string]int `json:"score_details"` + ScoreDetails string `json:"score_details"` } func GetCapacities(u *Unit, id string) (res map[string]int64) { @@ -246,22 +246,6 @@ func GetCapacities(u *Unit, id string) (res map[string]int64) { return } -func GetScore(u *Unit, id string, details bool) string { - score := u.GetScore(id) - s := fmt.Sprintf("%v", score) - if details { - scores := u.GetScores(id) - if len(scores) > 0 { - ss := []string{} - for name, score := range scores { - ss = append(ss, fmt.Sprintf("%v:%v", name, score)) - } - s += " (" + strings.Join(ss, ", ") + ")" - } - } - return s -} - type SchedResultItemList struct { Unit *Unit Data []*SchedResultItem @@ -325,16 +309,17 @@ func SelectHosts(unit *Unit, priorityList HostPriorityList) ([]*SelectedCandidat return nil, fmt.Errorf("SelectHosts get empty priorityList.") } - sort.Sort(sort.Reverse(priorityList)) - selectedMap := make(map[string]*SelectedCandidate) schedData := unit.SchedData() count := schedData.Count isSuggestion := unit.SchedInfo.IsSuggestion bestEffort := unit.SchedInfo.BestEffort selectedCandidates := []*SelectedCandidate{} + plugins := unit.AllSelectPlugins() + sort.Sort(sort.Reverse(priorityList)) + completed: for len(priorityList) > 0 { log.V(10).Debugf("PriorityList: %#v", priorityList) @@ -357,18 +342,8 @@ completed: } selectedItem.Count++ count-- - doPlugins := func() bool { - r := true - for _, plugin := range plugins { - if !plugin.OnSelect(unit, selectedItem.Candidate) { - r = false - } - } - return r - } - // if no one of plugins return false or capacity of the host large than - // selected count, this host can be added to priorityList. - if doPlugins() && unit.GetCapacity(hostID) > selectedItem.Count { + // if capacity of the host large than selected count, this host can be added to priorityList. + if unit.GetCapacity(hostID) > selectedItem.Count { priorityList0 = append(priorityList0, it) } } @@ -635,15 +610,23 @@ func PrioritizeCandidates( result := make(HostPriorityList, 0, len(candidates)) // TODO: Consider parallelizing it - for i := range candidates { - result = append(result, HostPriority{Host: candidates[i].IndexKey(), Score: 0, Candidate: candidates[i]}) - for j := range newPriorities { - result[i].Score += results[j][i].Score * newPriorities[j].Weight + // Do plugin priorities step + for _, candidate := range candidates { + for _, plugin := range unit.AllSelectPlugins() { + plugin.OnPriorityEnd(unit, candidate) } } + + for i, candidate := range candidates { + result = append(result, HostPriority{Host: candidates[i].IndexKey(), Score: *newScore(), Candidate: candidates[i]}) + //for j := range newPriorities { + //result[i].Score += results[j][i].Score * newPriorities[j].Weight + //} + result[i].Score = unit.GetScore(candidate.IndexKey()) + } if log.V(10) { for i := range result { - log.Infof("Host %s => Score %d", result[i].Host, result[i].Score) + log.Infof("Host %s => Score %s", result[i].Host, result[i].Score.DigitString()) } } return result, nil @@ -671,7 +654,7 @@ func EqualPriority(_ *Unit, candidate Candidater) (HostPriority, error) { } return HostPriority{ Host: indexKey, - Score: 1, + Score: newZeroScore(), Candidate: candidate, }, nil } diff --git a/pkg/scheduler/core/score/score.go b/pkg/scheduler/core/score/score.go new file mode 100644 index 0000000000..86c9475eaa --- /dev/null +++ b/pkg/scheduler/core/score/score.go @@ -0,0 +1,294 @@ +package score + +import ( + "container/list" + "fmt" + "math" + //"yunion.io/x/log" +) + +type TScore int + +const ( + MinScore TScore = -1 + ZeroScore TScore = 0 + MidScore TScore = 1 + MaxScore TScore = 2 + + ZeroScoreName = "zero" +) + +type SScore struct { + Score TScore + Name string +} + +func NewScore(score TScore, name string) SScore { + return SScore{ + Score: score, + Name: name, + } +} + +func NewMinScore(name string) SScore { + return NewScore(MinScore, name) +} + +func NewZeroScore() SScore { + return NewScore(ZeroScore, ZeroScoreName) +} + +func NewMidScore(name string) SScore { + return NewScore(MidScore, name) +} + +func NewMaxScore(name string) SScore { + return NewScore(MaxScore, name) +} + +func (v SScore) GetScore() TScore { + return v.Score +} + +func (v SScore) String() string { + return fmt.Sprintf("%s: %d", v.Name, v.Score) +} + +type Scores struct { + scores *list.List +} + +func newScores() *Scores { + return &Scores{ + scores: list.New(), + } +} + +func (s *Scores) Append(scores ...SScore) *Scores { + for _, score := range scores { + s.scores.PushBack(score) + } + return s +} + +func (s *Scores) AddToFirst(score SScore) *Scores { + s.scores.PushFront(score) + return s +} + +func (s *Scores) Range(iterFunc func(ele *list.Element, score SScore) bool) { + for ele := s.scores.Front(); ele != nil; ele = ele.Next() { + cont := iterFunc(ele, ele.Value.(SScore)) + if !cont { + break + } + } +} + +func (s *Scores) SetScore(score SScore) *Scores { + exists := false + rf := func(ele *list.Element, oscore SScore) bool { + if oscore.Name == score.Name { + exists = true + oscore.Score = score.Score + ele.Value = oscore + return false + } + return true + } + s.Range(rf) + if !exists { + s.Append(score) + } + return s +} + +func (s *Scores) AddScore(score SScore) *Scores { + exists := false + rf := func(ele *list.Element, oscore SScore) bool { + if oscore.Name == score.Name { + exists = true + oscore.Score += score.Score + ele.Value = oscore + return false + } + return true + } + s.Range(rf) + if !exists { + s.Append(score) + } + return s +} + +func (s *Scores) Len() int { + return s.scores.Len() +} + +func (s *Scores) GetScores() []SScore { + ret := make([]SScore, 0) + rf := func(_ *list.Element, score SScore) bool { + ret = append(ret, score) + return true + } + s.Range(rf) + return ret +} + +type ScoreBucket struct { + scores *Scores +} + +func NewScoreBuckets() *ScoreBucket { + return &ScoreBucket{ + scores: newScores(), + } +} + +func (b *ScoreBucket) AddToFirst(score SScore) *ScoreBucket { + b.scores.AddToFirst(score) + return b +} + +func (b *ScoreBucket) Append(scores ...SScore) *ScoreBucket { + b.scores.Append(scores...) + return b +} + +func (b *ScoreBucket) GetScores() []SScore { + return b.scores.GetScores() +} + +func (b *ScoreBucket) SetScore(score SScore) *ScoreBucket { + b.scores.SetScore(score) + return b +} + +func (b *ScoreBucket) AddScore(score SScore) *ScoreBucket { + b.scores.AddScore(score) + return b +} + +func (b *ScoreBucket) GetScore(scoreName string) (int, SScore) { + for i, oscore := range b.scores.GetScores() { + if oscore.Name == scoreName { + return i, oscore + } + } + return -1, SScore{} +} + +func (b *ScoreBucket) Len() int { + return b.scores.Len() +} + +func (b *ScoreBucket) DigitString() string { + s := "" + rf := func(_ *list.Element, score SScore) bool { + s = fmt.Sprintf("%s%d", s, score.Score) + return true + } + b.scores.Range(rf) + return s +} + +func extend(scores []SScore, length int) []SScore { + olen := len(scores) + if olen >= length { + return scores + } + ret := make([]SScore, 0) + zeroDigits := length - olen + for i := 0; i < zeroDigits; i++ { + ret = append(ret, NewZeroScore()) + } + ret = append(ret, scores...) + return ret +} + +func Equal(b1, b2 *ScoreBucket) bool { + return compare(b1, b2, func(s1, s2 TScore) bool { return s1 == s2 }) +} + +func Less(b1, b2 *ScoreBucket) bool { + return compare(b1, b2, func(s1, s2 TScore) bool { return s1 < s2 }) +} + +func compare(b1, b2 *ScoreBucket, cf func(s1, s2 TScore) bool) bool { + maxLen := int(math.Max(float64(b1.Len()), float64(b2.Len()))) + s1 := b1.GetScores() + s2 := b2.GetScores() + s1 = extend(s1, maxLen) + s2 = extend(s2, maxLen) + for i := range s1 { + v1 := s1[i].GetScore() + v2 := s2[i].GetScore() + ok := cf(v1, v2) + if ok { + return true + } else if !ok { + return false + } + } + return false +} + +func (b *ScoreBucket) debugString(vals []SScore, ret string) string { + if len(vals) == 0 { + return ret + } + restVal := vals[1:] + if len(restVal) == 0 { + return vals[0].String() + } + str := b.debugString(restVal, ret) + str = fmt.Sprintf("%s, %s", vals[0].String(), str) + return str +} + +func (b *ScoreBucket) String() string { + return b.debugString(b.GetScores(), "") +} + +type Interval struct { + start int64 + end int64 +} + +func NewInterval(start, end int64) *Interval { + return &Interval{start: start, end: end} +} + +func (i Interval) IsContain(val int64) bool { + return val >= i.start && val < i.end +} + +type Intervals struct { + MinInterval *Interval + ZeroInterval *Interval + MidInterval *Interval + MaxInterval *Interval +} + +func NewIntervals(min, zero, mid int64) Intervals { + return Intervals{ + MinInterval: NewInterval(math.MinInt64, min), + ZeroInterval: NewInterval(min, zero), + MidInterval: NewInterval(zero, mid), + MaxInterval: NewInterval(mid, math.MaxInt64), + } +} + +func (is Intervals) ToScore(val int64) TScore { + for score, interval := range map[TScore]*Interval{ + MinScore: is.MinInterval, + ZeroScore: is.ZeroInterval, + MidScore: is.MidInterval, + MaxScore: is.MaxInterval, + } { + if interval != nil && interval.IsContain(val) { + return score + } + } + return ZeroScore +} diff --git a/pkg/scheduler/core/score/score_test.go b/pkg/scheduler/core/score/score_test.go new file mode 100644 index 0000000000..b67c90d258 --- /dev/null +++ b/pkg/scheduler/core/score/score_test.go @@ -0,0 +1,156 @@ +package score + +import ( + "testing" +) + +func TestScoreBucket_String(t *testing.T) { + type fields struct { + scores *Scores + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "EmptyScores", + fields: fields{newScores()}, + want: "", + }, + { + name: "Scores100", + fields: fields{newScores().Append( + NewMidScore("mid"), + NewZeroScore(), + NewZeroScore(), + )}, + want: "mid: 1, zero: 0, zero: 0", + }, + { + name: "Scores201-1", + fields: fields{newScores().Append( + NewMaxScore("max"), + NewZeroScore(), + NewMidScore("mid"), + NewMinScore("min"), + )}, + want: "max: 2, zero: 0, mid: 1, min: -1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &ScoreBucket{ + scores: tt.fields.scores, + } + if got := b.String(); got != tt.want { + t.Errorf("ScoreBucket.String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestLess(t *testing.T) { + type args struct { + b1 *ScoreBucket + b2 *ScoreBucket + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "equal", + args: args{ + b1: NewScoreBuckets(), + b2: NewScoreBuckets(), + }, + want: false, + }, + { + name: "extendEqual", + args: args{ + b1: NewScoreBuckets().Append( + NewZeroScore(), NewMidScore("1"), + ), + b2: NewScoreBuckets().Append(NewMidScore("1")), + }, + want: false, + }, + { + name: "10<100", + args: args{ + b1: NewScoreBuckets().Append( + NewMidScore("1"), + NewZeroScore(), + ), + b2: NewScoreBuckets().Append( + NewMidScore("1"), + NewZeroScore(), + NewZeroScore(), + ), + }, + want: true, + }, + { + name: "101>10", + args: args{ + b1: NewScoreBuckets().Append( + NewMidScore("1"), + NewZeroScore(), + NewMidScore("1"), + ), + b2: NewScoreBuckets().Append( + NewMidScore("1"), + NewZeroScore(), + ), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Less(tt.args.b1, tt.args.b2); got != tt.want { + t.Errorf("Less() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestScoreBucket_DigitString(t *testing.T) { + type fields struct { + scores *Scores + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "2-101", + fields: fields{newScores().Append( + NewMaxScore(""), + NewMinScore(""), + NewZeroScore(), + NewMidScore(""), + )}, + want: "2-101", + }, + { + name: "empty", + fields: fields{newScores()}, + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &ScoreBucket{ + scores: tt.fields.scores, + } + if got := b.DigitString(); got != tt.want { + t.Errorf("ScoreBucket.DigitString() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 7ca0459fc8..7deb8724b4 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -1,15 +1,14 @@ package core import ( - //"sync" - //"yunion.io/x/onecloud/pkg/scheduler/cache/candidate" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/scheduler/core/score" "yunion.io/x/onecloud/pkg/scheduler/db/models" ) const ( - PriorityStep int = 100 + PriorityStep int = 1 ) type FailedCandidate struct { @@ -23,7 +22,8 @@ type FailedCandidates struct { } type SelectPlugin interface { - OnSelect(*Unit, Candidater) bool + Name() string + OnPriorityEnd(*Unit, Candidater) OnSelectEnd(u *Unit, c Candidater, count int64) } @@ -58,7 +58,7 @@ type HostPriority struct { // Name of the host Host string // Score associated with the host - Score int + Score Score // Resource wraps Candidate host info Candidate Candidater } @@ -70,10 +70,10 @@ func (h HostPriorityList) Len() int { } func (h HostPriorityList) Less(i, j int) bool { - if h[i].Score == h[j].Score { + if score.Equal(h[i].Score.ScoreBucket, h[j].Score.ScoreBucket) { return h[i].Host < h[j].Host } - return h[i].Score < h[j].Score + return score.Less(h[i].Score.ScoreBucket, h[j].Score.ScoreBucket) } func (h HostPriorityList) Swap(i, j int) { @@ -115,4 +115,7 @@ type Priority interface { Map(*Unit, Candidater) (HostPriority, error) Reduce(*Unit, []Candidater, HostPriorityList) error PreExecute(*Unit, []Candidater) (bool, []PredicateFailureReason, error) + + // Score intervals + ScoreIntervals() score.Intervals } diff --git a/pkg/util/hashcache/doc.go b/pkg/util/hashcache/doc.go new file mode 100644 index 0000000000..50f6faf063 --- /dev/null +++ b/pkg/util/hashcache/doc.go @@ -0,0 +1 @@ +package hashcache // import "yunion.io/x/onecloud/pkg/util/hashcache" From 9a80956ffa9875dc72bf37b0f01e2e2d8b6a36d3 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 12 Dec 2018 20:19:54 +0800 Subject: [PATCH 24/34] climc: make scheduler test project work --- cmd/climc/shell/schedulers.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/climc/shell/schedulers.go b/cmd/climc/shell/schedulers.go index 47cd7210cf..3da9b11dfd 100644 --- a/cmd/climc/shell/schedulers.go +++ b/cmd/climc/shell/schedulers.go @@ -66,7 +66,15 @@ func init() { } } if len(args.Project) > 0 { - data.Add(jsonutils.NewString(args.Project), "tenant") + ret, err := modules.Projects.Get(s, args.Project, nil) + if err != nil { + return err + } + projectId, err := ret.GetString("id") + if err != nil { + return err + } + data.Add(jsonutils.NewString(projectId), "owner_tenant_id") } if len(args.Hypervisor) > 0 { data.Add(jsonutils.NewString(args.Hypervisor), "hypervisor") From 5f3d45201715f2a34b58e86836f67edd202e489a Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 12 Dec 2018 20:51:18 +0800 Subject: [PATCH 25/34] =?UTF-8?q?region:=20server=20deploy=20=E5=92=8C=20s?= =?UTF-8?q?ave-image=20=E6=93=8D=E4=BD=9C=E5=90=8E=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E9=87=8D=E5=90=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/guestdrivers/kvm.go | 2 +- pkg/compute/models/guest_actions.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 1cb9d0d586..7e94be6da8 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -273,7 +273,7 @@ func (self *SKVMGuestDriver) GetChangeConfigStatus() ([]string, error) { } func (self *SKVMGuestDriver) GetDeployStatus() ([]string, error) { - return []string{models.VM_READY}, nil + return []string{models.VM_READY, models.VM_RUNNING, models.VM_ADMIN}, nil } func (self *SKVMGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 820658c7e0..fbf09a6f4c 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -91,7 +91,7 @@ func (self *SGuest) PerformSaveImage(ctx context.Context, userCred mcclient.Toke return nil, httperrors.NewInputParameterError("No root image") } else { kwargs := data.(*jsonutils.JSONDict) - restart := self.Status == VM_RUNNING + restart := (self.Status == VM_RUNNING) || jsonutils.QueryBoolean(data, "auto_start", false) properties := jsonutils.NewDict() if notes, err := data.GetString("notes"); err != nil && len(notes) > 0 { properties.Add(jsonutils.NewString(notes), "notes") @@ -334,7 +334,8 @@ func (self *SGuest) PerformDeploy(ctx context.Context, userCred mcclient.TokenCr } if utils.IsInStringArray(self.Status, deployStatus) { - if doRestart && self.Status == VM_RUNNING { + if (doRestart && self.Status == VM_RUNNING) || + jsonutils.QueryBoolean(kwargs, "auto_start", false) { kwargs.Set("restart", jsonutils.JSONTrue) } err := self.StartGuestDeployTask(ctx, userCred, kwargs, "deploy", "") From 33e0269d081ab3a0afe7b2501ec032b502ef95ec Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 21:01:02 +0800 Subject: [PATCH 26/34] minor fixes --- pkg/compute/models/host_recycle.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/compute/models/host_recycle.go b/pkg/compute/models/host_recycle.go index 4b2284df4a..90060167c2 100644 --- a/pkg/compute/models/host_recycle.go +++ b/pkg/compute/models/host_recycle.go @@ -321,6 +321,10 @@ func (self *SHost) PerformUndoPrepaidRecycle(ctx context.Context, userCred mccli return nil, httperrors.NewInvalidStatusError("a recycle host shoud not allocate more than 1 guest") } + if guests[0].PendingDeleted { + return nil, httperrors.NewInvalidStatusError("cannot undo a recycle host with pending_deleted guest") + } + err := doUndoPrepaidRecycle(ctx, userCred, self, &guests[0]) if err != nil { logclient.AddActionLog(self, logclient.ACT_UNDO_RECYCLE_PREPAID, self.GetShortDesc(), userCred, false) @@ -584,7 +588,11 @@ func (host *SHost) RebuildRecycledGuest(ctx context.Context, userCred mcclient.T q := HostManager.Query() q = q.Equals("external_id", host.ExternalId) - q = q.IsNullOrEmpty("resource_type") + q = q.Filter(sqlchemy.OR( + sqlchemy.IsNullOrEmpty(q.Field("resource_type")), + sqlchemy.Equals(q.Field("resource_type"), HostResourceTypeShared), + )) + err := q.First(&oHost) if err != nil { log.Errorf("query oHost fail %s", err) From 9473893d77e51ca7af7db607b7b154f36063cf77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 12 Dec 2018 21:37:03 +0800 Subject: [PATCH 27/34] =?UTF-8?q?=E5=86=B2=E7=AA=81=E8=A7=A3=E5=86=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/mcclient/options/servers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index c3725d5d18..172c7d1d27 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -265,7 +265,7 @@ type ServerSecGroupOptions struct { type ServerSecGroupsOptions struct { ID string `help:"ID or Name of server" metavar:"Guest" json:"-"` - Secgrps []string `help:"Ids of Security Group" metavar:"Security Group" positional:"true"` + Secgrps []string `help:"Ids of Security Groups" metavar:"Security Groups" positional:"true"` } type ServerSendKeyOptions struct { From b736983fe9f770348e8b90169bd3d90e3ed07038 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 13:18:22 +0000 Subject: [PATCH 28/34] =?UTF-8?q?routetables:=20routetable-list=20--manage?= =?UTF-8?q?r=20=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/routetables.go | 14 +++++++------- pkg/mcclient/options/routetables.go | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/compute/models/routetables.go b/pkg/compute/models/routetables.go index ebf7930615..07eda270c9 100644 --- a/pkg/compute/models/routetables.go +++ b/pkg/compute/models/routetables.go @@ -108,13 +108,13 @@ func (man *SRouteTableManager) ListItemFilter(ctx context.Context, q *sqlchemy.S } userProjId := userCred.GetProjectId() data := query.(*jsonutils.JSONDict) - for _, key := range []string{"vpc", "cloudregion"} { - v := validators.NewModelIdOrNameValidator(key, key, userProjId) - v.Optional(true) - q, err = v.QueryFilter(q, data) - if err != nil { - return nil, err - } + q, err = validators.ApplyModelFilters(q, data, []*validators.ModelFilterOptions{ + {Key: "vpc", ModelKeyword: "vpc", ProjectId: userProjId}, + {Key: "cloudregion", ModelKeyword: "cloudregion", ProjectId: userProjId}, + {Key: "manager", ModelKeyword: "cloudprovider", ProjectId: userProjId}, + }) + if err != nil { + return nil, err } return q, nil } diff --git a/pkg/mcclient/options/routetables.go b/pkg/mcclient/options/routetables.go index c214d824b8..004aaf20c3 100644 --- a/pkg/mcclient/options/routetables.go +++ b/pkg/mcclient/options/routetables.go @@ -126,6 +126,7 @@ type RouteTableDeleteOptions struct { } type RouteTableListOptions struct { + Manager string Vpc string Cloudregion string From 010cafa4c5c81b895bc14b92448b5b5220528aa0 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 13:14:11 +0000 Subject: [PATCH 29/34] =?UTF-8?q?routetables:=20routetable-purge=E6=94=AF?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/routetables.go | 8 ++++++++ pkg/compute/models/routetables.go | 23 +++++++++++++++++++++++ pkg/mcclient/options/routetables.go | 4 ++++ 3 files changed, 35 insertions(+) diff --git a/cmd/climc/shell/routetables.go b/cmd/climc/shell/routetables.go index 32f0207830..32407e0736 100644 --- a/cmd/climc/shell/routetables.go +++ b/cmd/climc/shell/routetables.go @@ -114,4 +114,12 @@ func init() { printObjectRecursive(routetable) return nil }) + R(&options.RouteTablePurgeOptions{}, "routetable-purge", "Purge routetable", func(s *mcclient.ClientSession, opts *options.RouteTablePurgeOptions) error { + routetable, err := modules.RouteTables.PerformAction(s, opts.ID, "purge", nil) + if err != nil { + return err + } + printObjectRecursive(routetable) + return nil + }) } diff --git a/pkg/compute/models/routetables.go b/pkg/compute/models/routetables.go index 07eda270c9..95774d9ae8 100644 --- a/pkg/compute/models/routetables.go +++ b/pkg/compute/models/routetables.go @@ -150,6 +150,29 @@ func (man *SRouteTableManager) ValidateCreateData(ctx context.Context, userCred return man.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data) } +func (rt *SRouteTable) AllowPerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return db.IsAdminAllowPerform(userCred, rt, "purge") +} + +func (rt *SRouteTable) PerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + err := rt.ValidateDeleteCondition(ctx) + if err != nil { + return nil, err + } + provider := rt.GetCloudprovider() + if provider != nil { + if provider.Enabled { + return nil, httperrors.NewInvalidStatusError("Cannot purge route_table on enabled cloud provider") + } + } + err = rt.RealDelete(ctx, userCred) + return nil, err +} + +func (rt *SRouteTable) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return rt.SVirtualResourceBase.Delete(ctx, userCred) +} + func (rt *SRouteTable) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { data, err := RouteTableManager.validateRoutes(data, true) if err != nil { diff --git a/pkg/mcclient/options/routetables.go b/pkg/mcclient/options/routetables.go index 004aaf20c3..23e1d810bb 100644 --- a/pkg/mcclient/options/routetables.go +++ b/pkg/mcclient/options/routetables.go @@ -125,6 +125,10 @@ type RouteTableDeleteOptions struct { ID string } +type RouteTablePurgeOptions struct { + ID string +} + type RouteTableListOptions struct { Manager string Vpc string From 91f61d6b7e2397839a38ac8e14f681e3e275abb3 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 13:49:50 +0000 Subject: [PATCH 30/34] =?UTF-8?q?guests:=20=E5=85=81=E8=AE=B8purge?= =?UTF-8?q?=E9=A2=84=E4=BB=98=E8=B4=B9=E8=99=9A=E6=9C=BA=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/compute/models/guest_actions.go | 2 +- pkg/compute/models/guests.go | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 2742b69371..d2990b3a6d 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -771,7 +771,7 @@ func (self *SGuest) AllowPerformPurge(ctx context.Context, userCred mcclient.Tok } func (self *SGuest) PerformPurge(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - err := self.ValidateDeleteCondition(ctx) + err := self.ValidatePurgeCondition(ctx) if err != nil { return nil, err } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 784bc461b9..1c83f2e5ec 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -481,16 +481,24 @@ func (guest *SGuest) GetDriver() IGuestDriver { return GetDriver(hypervisor) } -func (guest *SGuest) ValidateDeleteCondition(ctx context.Context) error { +func (guest *SGuest) validateDeleteCondition(ctx context.Context, isPurge bool) error { if guest.DisableDelete.IsTrue() { return httperrors.NewInvalidStatusError("Virtual server is locked, cannot delete") } - if guest.IsValidPrePaid() { + if !isPurge && guest.IsValidPrePaid() { return httperrors.NewForbiddenError("not allow to delete prepaid server in valid status") } return guest.SVirtualResourceBase.ValidateDeleteCondition(ctx) } +func (guest *SGuest) ValidatePurgeCondition(ctx context.Context) error { + return guest.validateDeleteCondition(ctx, true) +} + +func (guest *SGuest) ValidateDeleteCondition(ctx context.Context) error { + return guest.validateDeleteCondition(ctx, false) +} + func (guest *SGuest) GetDisksQuery() *sqlchemy.SQuery { return GuestdiskManager.Query().Equals("guest_id", guest.Id) } From 8f11b765d9c3f3f76ec8a007c1707c453be89ef3 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 22:53:13 +0800 Subject: [PATCH 31/34] =?UTF-8?q?1.=20=E5=AF=B9disk/elasticip/networks/sna?= =?UTF-8?q?pshot/storages/vpc/wire/routetable/servers=E7=AD=89=E6=94=AF?= =?UTF-8?q?=E6=8C=81manager/account/provider=E8=BF=87=E6=BB=A4=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=202.=20=E8=AF=A6=E6=83=85=E5=A2=9E=E5=8A=A0SCloudProv?= =?UTF-8?q?iderInfo=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/shell/disks.go | 4 +- cmd/climc/shell/elasticips.go | 15 ++++- cmd/climc/shell/networks.go | 13 ++++ cmd/climc/shell/snapshots.go | 8 ++- cmd/climc/shell/storages.go | 10 ++- cmd/climc/shell/vpcs.go | 14 +++- cmd/climc/shell/wires.go | 23 ++++++- pkg/appsrv/handlerinfo.go | 2 +- pkg/cloudprovider/consts.go | 1 + pkg/compute/handlers.go | 1 + pkg/compute/models/billingresource.go | 85 ++++-------------------- pkg/compute/models/disks.go | 62 +++++++++++++---- pkg/compute/models/elasticips.go | 27 +++++++- pkg/compute/models/guests.go | 43 +++++++++++- pkg/compute/models/hosts.go | 14 ++-- pkg/compute/models/managedresource.go | 78 +++++++++++++++++++++- pkg/compute/models/networks.go | 69 ++++++++++++++++++- pkg/compute/models/routetables.go | 78 +++++++++++++++++++++- pkg/compute/models/snapshots.go | 36 +++++++--- pkg/compute/models/storages.go | 46 +++++++++++-- pkg/compute/models/vpcs.go | 58 +++++++++++++--- pkg/compute/models/wires.go | 53 ++++++++++++++- pkg/compute/tasks/eip_deallocate_task.go | 2 +- pkg/mcclient/options/routetables.go | 4 ++ pkg/mcclient/options/servers.go | 5 +- 25 files changed, 603 insertions(+), 148 deletions(-) diff --git a/cmd/climc/shell/disks.go b/cmd/climc/shell/disks.go index 98621e02ca..9e616673ec 100644 --- a/cmd/climc/shell/disks.go +++ b/cmd/climc/shell/disks.go @@ -18,7 +18,9 @@ func init() { Guest string `help:"Guest ID or name"` Storage string `help:"Storage ID or name"` - Provider string `help:"Provider for disk" choices:"Aliyun|VMware|Azure"` + Manager string `help:"List disks belongs to the cloud provider"` + Account string `help:"List disks belongs to the cloud account"` + Provider string `help:"List disks belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` BillingType string `help:"billing type" choices:"postpaid|prepaid"` } diff --git a/cmd/climc/shell/elasticips.go b/cmd/climc/shell/elasticips.go index 74e6e0f281..7e72b5fd8a 100644 --- a/cmd/climc/shell/elasticips.go +++ b/cmd/climc/shell/elasticips.go @@ -10,9 +10,12 @@ import ( func init() { type ElasticipListOptions struct { - Manager string `help:"Show servers imported from manager"` - Region string `help:"Show servers in cloudregion"` - Usable bool `help:"List all zones that is usable"` + Manager string `help:"Show servers imported from manager"` + Region string `help:"Show servers in cloudregion"` + Account string `help:"List hosts belongs to the cloud account"` + Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` + + Usable bool `help:"List all zones that is usable"` options.BaseListOptions } @@ -28,6 +31,12 @@ func init() { if len(args.Manager) > 0 { params.Add(jsonutils.NewString(args.Manager), "manager") } + if len(args.Account) > 0 { + params.Add(jsonutils.NewString(args.Account), "account") + } + if len(args.Provider) > 0 { + params.Add(jsonutils.NewString(args.Provider), "provider") + } if len(args.Region) > 0 { params.Add(jsonutils.NewString(args.Region), "region") } diff --git a/cmd/climc/shell/networks.go b/cmd/climc/shell/networks.go index 28688112bd..a4ebfd6eb1 100644 --- a/cmd/climc/shell/networks.go +++ b/cmd/climc/shell/networks.go @@ -18,6 +18,10 @@ func init() { Vpc string `help:"search networks belongs to a VPC"` Region string `help:"search networks belongs to a CloudRegion"` ServerType string `help:"search networks belongs to a ServerType"` + + Manager string `help:"List networks belongs to the cloud provider"` + Account string `help:"List networks belongs to the cloud account"` + Provider string `help:"List networks belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` } R(&NetworkListOptions{}, "network-list", "List networks", func(s *mcclient.ClientSession, args *NetworkListOptions) error { var params *jsonutils.JSONDict @@ -44,6 +48,15 @@ func init() { if len(args.ServerType) > 0 { params.Add(jsonutils.NewString(args.ServerType), "server_type") } + if len(args.Manager) > 0 { + params.Add(jsonutils.NewString(args.Manager), "manager") + } + if len(args.Account) > 0 { + params.Add(jsonutils.NewString(args.Account), "account") + } + if len(args.Provider) > 0 { + params.Add(jsonutils.NewString(args.Provider), "provider") + } var result *modules.ListResult var err error if len(args.Wire) > 0 { diff --git a/cmd/climc/shell/snapshots.go b/cmd/climc/shell/snapshots.go index cf63c3be9a..8bbd4d2459 100644 --- a/cmd/climc/shell/snapshots.go +++ b/cmd/climc/shell/snapshots.go @@ -15,9 +15,10 @@ func init() { Local bool `help:"Show local snapshots"` Share bool `help:"Show shared snapshots"` DiskType string `help:"Filter by disk type" choices:"sys|data"` - Provider string `help:"Cloud provider" choices:"Aliyun|VMware|Azure"` - Manager string `help:"Show snapshots belongs to a specific cloud provider"` + Manager string `help:"Show snapshots belongs to a specific cloud provider"` + Account string `help:"List hosts belongs to the cloud account"` + Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` } R(&SnapshotsListOptions{}, "snapshot-list", "Show snapshots", func(s *mcclient.ClientSession, args *SnapshotsListOptions) error { params, err := args.BaseListOptions.Params() @@ -46,6 +47,9 @@ func init() { if len(args.Manager) > 0 { params.Add(jsonutils.NewString(args.Manager), "manager") } + if len(args.Account) > 0 { + params.Add(jsonutils.NewString(args.Account), "account") + } result, err := modules.Snapshots.List(s, params) if err != nil { return err diff --git a/cmd/climc/shell/storages.go b/cmd/climc/shell/storages.go index 5392b09073..8929f0fade 100644 --- a/cmd/climc/shell/storages.go +++ b/cmd/climc/shell/storages.go @@ -18,7 +18,9 @@ func init() { Zone string `help:"List storages in zone"` Region string `help:"List storages in region"` - Manager string `help:"Show regions belongs to the cloud provider"` + Manager string `help:"List storages belongs to the cloud provider"` + Account string `help:"List storages belongs to the cloud account"` + Provider string `help:"List storages belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` } R(&StorageListOptions{}, "storage-list", "List storages", func(s *mcclient.ClientSession, args *StorageListOptions) error { var params *jsonutils.JSONDict @@ -46,6 +48,12 @@ func init() { if len(args.Manager) > 0 { params.Add(jsonutils.NewString(args.Manager), "manager") } + if len(args.Account) > 0 { + params.Add(jsonutils.NewString(args.Account), "account") + } + if len(args.Provider) > 0 { + params.Add(jsonutils.NewString(args.Provider), "provider") + } var result *modules.ListResult var err error diff --git a/cmd/climc/shell/vpcs.go b/cmd/climc/shell/vpcs.go index 79517f2880..1a7c34426f 100644 --- a/cmd/climc/shell/vpcs.go +++ b/cmd/climc/shell/vpcs.go @@ -10,8 +10,12 @@ import ( func init() { type VpcListOptions struct { options.BaseListOptions - Region string `help:"ID or Name of region"` - Manager string `help:"Show vpcs belongs to the cloud provider"` + + Region string `help:"ID or Name of region"` + + Manager string `help:"List vpcs belongs to the cloud provider"` + Account string `help:"List vpcs belongs to the cloud account"` + Provider string `help:"List vpcs belongs to the public cloud" choices:"Aliyun|Qcloud|Azure|Aws|Huawei"` } R(&VpcListOptions{}, "vpc-list", "List VPCs", func(s *mcclient.ClientSession, args *VpcListOptions) error { var params *jsonutils.JSONDict @@ -27,6 +31,12 @@ func init() { if len(args.Manager) > 0 { params.Add(jsonutils.NewString(args.Manager), "manager") } + if len(args.Account) > 0 { + params.Add(jsonutils.NewString(args.Account), "account") + } + if len(args.Provider) > 0 { + params.Add(jsonutils.NewString(args.Provider), "provider") + } var result *modules.ListResult var err error diff --git a/cmd/climc/shell/wires.go b/cmd/climc/shell/wires.go index d83d778cc0..00f0085eab 100644 --- a/cmd/climc/shell/wires.go +++ b/cmd/climc/shell/wires.go @@ -10,8 +10,14 @@ import ( func init() { type WireListOptions struct { options.BaseListOptions - Zone string `help:"list wires in zone"` - Vpc string `help:"List wires in vpc"` + + Region string `help:"List hosts in region"` + Zone string `help:"list wires in zone"` + Vpc string `help:"List wires in vpc"` + + Manager string `help:"List hosts belongs to the cloud provider"` + Account string `help:"List hosts belongs to the cloud account"` + Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` } R(&WireListOptions{}, "wire-list", "List wires", func(s *mcclient.ClientSession, args *WireListOptions) error { var params *jsonutils.JSONDict @@ -26,6 +32,19 @@ func init() { if len(args.Vpc) > 0 { params.Add(jsonutils.NewString(args.Vpc), "vpc") } + if len(args.Region) > 0 { + params.Add(jsonutils.NewString(args.Region), "region") + } + if len(args.Manager) > 0 { + params.Add(jsonutils.NewString(args.Manager), "manager") + } + if len(args.Account) > 0 { + params.Add(jsonutils.NewString(args.Account), "account") + } + if len(args.Provider) > 0 { + params.Add(jsonutils.NewString(args.Provider), "provider") + } + var result *modules.ListResult var err error if len(args.Zone) > 0 { diff --git a/pkg/appsrv/handlerinfo.go b/pkg/appsrv/handlerinfo.go index 6a7234ca65..048a4124cd 100644 --- a/pkg/appsrv/handlerinfo.go +++ b/pkg/appsrv/handlerinfo.go @@ -110,4 +110,4 @@ func (hi *SHandlerInfo) GetAppParams(params map[string]string, path []string) *S appParams.Params = params appParams.Path = path return &appParams -} \ No newline at end of file +} diff --git a/pkg/cloudprovider/consts.go b/pkg/cloudprovider/consts.go index d8fde2e064..a4c7137159 100644 --- a/pkg/cloudprovider/consts.go +++ b/pkg/cloudprovider/consts.go @@ -17,3 +17,4 @@ var ErrInvalidStatus = errors.New("invalid status") var ErrTimeout = errors.New("timeout") var ErrNotImplemented = errors.New("Not implemented") var ErrNotSupported = errors.New("Not supported") +var ErrInvalidProvider = errors.New("Invalid provider") diff --git a/pkg/compute/handlers.go b/pkg/compute/handlers.go index c55147719b..7f5a2ed114 100644 --- a/pkg/compute/handlers.go +++ b/pkg/compute/handlers.go @@ -34,6 +34,7 @@ func InitHandlers(app *appsrv.Application) { db.Metadata, models.GuestcdromManager, models.NetInterfaceManager, + models.VCenterManager, } { db.RegisterModelManager(manager) } diff --git a/pkg/compute/models/billingresource.go b/pkg/compute/models/billingresource.go index 69c575a207..c03c80666d 100644 --- a/pkg/compute/models/billingresource.go +++ b/pkg/compute/models/billingresource.go @@ -2,9 +2,6 @@ package models import ( "time" - - "yunion.io/x/onecloud/pkg/appctx" - "yunion.io/x/onecloud/pkg/cloudcommon/db" ) const ( @@ -26,12 +23,14 @@ func (self *SBillingResourceBase) GetChargeType() string { } } -func (self *SBillingResourceBase) FetchCloudBillingInfo(info *SCloudBillingInfo) { +func (self *SBillingResourceBase) getBillingBaseInfo() SBillingBaseInfo { + info := SBillingBaseInfo{} info.ChargeType = self.GetChargeType() if self.GetChargeType() == BILLING_TYPE_PREPAID { info.ExpiredAt = self.ExpiredAt info.BillingCycle = self.BillingCycle } + return info } func (self *SBillingResourceBase) IsValidPrePaid() bool { @@ -44,75 +43,17 @@ func (self *SBillingResourceBase) IsValidPrePaid() bool { return false } +type SBillingBaseInfo struct { + ChargeType string `json:",omitempty"` + ExpiredAt time.Time `json:",omitempty"` + BillingCycle string `json:",omitempty"` +} + type SCloudBillingInfo struct { - Provider string `json:",omitempty"` - Account string `json:",omitempty"` - AccountId string `json:",omitempty"` - Manager string `json:",omitempty"` - ManagerId string `json:",omitempty"` - ManagerProject string `json:",omitempty"` - ManagerProjectId string `json:",omitempty"` - Region string `json:",omitempty"` - RegionId string `json:",omitempty"` - RegionExtId string `json:",omitempty"` - Zone string `json:",omitempty"` - ZoneId string `json:",omitempty"` - ZoneExtId string `json:",omitempty"` - PriceKey string `json:",omitempty"` - ChargeType string `json:",omitempty"` - InternetChargeType string `json:",omitempty"` - ExpiredAt time.Time `json:",omitempty"` - BillingCycle string `json:",omitempty"` -} + SCloudProviderInfo -func MakeCloudBillingInfo(region *SCloudregion, zone *SZone, provider *SCloudprovider) SCloudBillingInfo { - info := SCloudBillingInfo{} + SBillingBaseInfo - if zone != nil { - info.Zone = zone.GetName() - info.ZoneId = zone.GetId() - } - - if region != nil { - info.Region = region.GetName() - info.RegionId = region.GetId() - } - - if provider != nil { - info.Manager = provider.GetName() - info.ManagerId = provider.GetId() - - if len(provider.ProjectId) > 0 { - info.ManagerProjectId = provider.ProjectId - tc, err := db.TenantCacheManager.FetchTenantById(appctx.Background, provider.ProjectId) - if err == nil { - info.ManagerProject = tc.GetName() - } - } - - account := provider.GetCloudaccount() - info.Account = account.GetName() - info.AccountId = account.GetId() - - driver, err := provider.GetDriver() - - if err == nil { - info.Provider = driver.GetId() - - if region != nil { - iregion, err := driver.GetIRegionById(region.ExternalId) - if err == nil { - info.RegionExtId = iregion.GetId() - if zone != nil { - izone, err := iregion.GetIZoneById(zone.ExternalId) - if err == nil { - info.ZoneExtId = izone.GetId() - } - } - } - } - } - } - - return info + PriceKey string `json:",omitempty"` + InternetChargeType string `json:",omitempty"` } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 1f43ae83e7..8053d04aa8 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -158,14 +158,7 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu sq := storages.Query(storages.Field("id")).Filter(sqlchemy.In(storages.Field("storage_type"), STORAGE_LOCAL_TYPES)) q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq)) } - if provier, _ := queryDict.GetString("provider"); len(provier) > 0 { - cloudprovider := CloudproviderManager.Query().SubQuery() - sq := storages.Query(storages.Field("id")).Join(cloudprovider, - sqlchemy.AND( - sqlchemy.Equals(cloudprovider.Field("id"), storages.Field("manager_id")), - sqlchemy.Equals(cloudprovider.Field("provider"), provier))) - q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq)) - } + guestId, _ := queryDict.GetString("guest") if len(guestId) != 0 { guest := GuestManager.FetchGuestById(guestId) @@ -190,6 +183,47 @@ func (manager *SDiskManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu } q = q.Filter(sqlchemy.Equals(q.Field("storage_id"), storageObj.GetId())) } + + managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"}) + if len(managerStr) > 0 { + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr) + } + return nil, httperrors.NewGeneralError(err) + } + subq := storages.Query(storages.Field("id")).Equals("manager_id", provider.GetId()) + q = q.Filter(sqlchemy.In(q.Field("storage_id"), subq.SubQuery())) + } + + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + + cloudproviders := CloudproviderManager.Query().SubQuery() + subq := storages.Query(storages.Field("id")) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), storages.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId())) + + q = q.Filter(sqlchemy.In(q.Field("storage_id"), subq.SubQuery())) + } + + if provier, _ := queryDict.GetString("provider"); len(provier) > 0 { + cloudproviders := CloudproviderManager.Query().SubQuery() + sq := storages.Query(storages.Field("id")) + sq = sq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), storages.Field("manager_id"))) + sq = sq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), provier)) + + q = q.Filter(sqlchemy.In(q.Field("storage_id"), sq.SubQuery())) + } + return q, nil } @@ -1202,15 +1236,19 @@ func (self *SDisk) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict extra.Add(jsonutils.NewString(storage.GetName()), "storage") extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") extra.Add(jsonutils.NewString(storage.MediumType), "medium_type") - extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") + /*extra.Add(jsonutils.NewString(storage.ZoneId), "zone_id") if zone := storage.getZone(); zone != nil { extra.Add(jsonutils.NewString(zone.Name), "zone") extra.Add(jsonutils.NewString(zone.CloudregionId), "region_id") if region := zone.GetRegion(); region != nil { extra.Add(jsonutils.NewString(region.Name), "region") } - } + }*/ + + info := storage.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) } + guests, guest_status := []string{}, []string{} for _, guest := range self.GetGuests() { guests = append(guests, guest.Name) @@ -1344,14 +1382,14 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict { var billingInfo SCloudBillingInfo if storage != nil { - billingInfo = storage.getCloudBillingInfo() + billingInfo.SCloudProviderInfo = storage.getCloudProviderInfo() } if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { billingInfo.PriceKey = priceKey } - self.FetchCloudBillingInfo(&billingInfo) + billingInfo.SBillingBaseInfo = self.getBillingBaseInfo() desc.Update(jsonutils.Marshal(billingInfo)) diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index 0bdcf5b7f3..c8dc5ad80a 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -114,6 +114,25 @@ func (manager *SElasticipManager) ListItemFilter(ctx context.Context, q *sqlchem q = q.Equals("cloudregion_id", regionObj.GetId()) } + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + } + if query.Contains("usable") { usable := jsonutils.QueryBoolean(query, "usable", false) if usable { @@ -159,7 +178,9 @@ func (self *SElasticip) GetShortDesc() *jsonutils.JSONDict { // } //} - billingInfo := self.getCloudBillingInfo() + billingInfo := SCloudBillingInfo{} + + billingInfo.SCloudProviderInfo = self.getCloudProviderInfo() billingInfo.InternetChargeType = self.ChargeType @@ -921,8 +942,8 @@ func (self *SElasticip) DoPendingDelete(ctx context.Context, userCred mcclient.T self.Dissociate(ctx, userCred) } -func (self *SElasticip) getCloudBillingInfo() SCloudBillingInfo { +func (self *SElasticip) getCloudProviderInfo() SCloudProviderInfo { region := self.GetRegion() provider := self.GetCloudprovider() - return MakeCloudBillingInfo(region, nil, provider) + return MakeCloudProviderInfo(region, nil, provider) } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 352012aa0b..69d45d396d 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -399,6 +399,37 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ q = q.In("host_id", sq) } + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + hosts := HostManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := hosts.Query(hosts.Field("id")) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), hosts.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId())) + + q = q.Filter(sqlchemy.In(q.Field("host_id"), subq.SubQuery())) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + hosts := HostManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := hosts.Query(hosts.Field("id")) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), hosts.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr)) + + q = q.Filter(sqlchemy.In(q.Field("host_id"), subq.SubQuery())) + } + regionFilter, _ := queryDict.GetString("region") if len(regionFilter) > 0 { regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionFilter) @@ -1113,7 +1144,7 @@ func (self *SGuest) GetCustomizeColumns(ctx context.Context, userCred mcclient.T } func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict { - zone := self.getZone() + /*zone := self.getZone() if zone != nil { extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") extra.Add(jsonutils.NewString(zone.GetName()), "zone") @@ -1139,6 +1170,12 @@ func (self *SGuest) moreExtraInfo(extra *jsonutils.JSONDict) *jsonutils.JSONDict extra.Add(jsonutils.NewString(provider.GetName()), "manager") } } + }*/ + + host := self.GetHost() + if host != nil { + info := host.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) } err := self.CanPerformPrepaidRecycle() @@ -3171,14 +3208,14 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { var billingInfo SCloudBillingInfo if host != nil { - billingInfo = host.getCloudBillingInfo() + billingInfo.SCloudProviderInfo = host.getCloudProviderInfo() } if priceKey := self.GetMetadata("price_key", nil); len(priceKey) > 0 { billingInfo.PriceKey = priceKey } - self.FetchCloudBillingInfo(&billingInfo) + billingInfo.SBillingBaseInfo = self.getBillingBaseInfo() desc.Update(jsonutils.Marshal(billingInfo)) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 5760927f1c..2d2fe4b7e5 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -35,7 +36,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/logclient" - "yunion.io/x/onecloud/pkg/appsrv" ) const ( @@ -1999,10 +1999,8 @@ func (self *SHost) getMoreDetails(ctx context.Context, extra *jsonutils.JSONDict } }*/ - info := self.getCloudBillingInfo() - infoJson := jsonutils.Marshal(&info) - log.Debugf("%s", infoJson.String()) - extra.Update(infoJson) + info := self.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) server := self.GetBaremetalServer() if server != nil { @@ -3500,17 +3498,17 @@ func (manager *SHostManager) GetHostByIp(hostIp string) (*SHost, error) { return host.(*SHost), nil } -func (self *SHost) getCloudBillingInfo() SCloudBillingInfo { +func (self *SHost) getCloudProviderInfo() SCloudProviderInfo { var region *SCloudregion zone := self.GetZone() if zone != nil { region = zone.GetRegion() } provider := self.GetCloudprovider() - return MakeCloudBillingInfo(region, zone, provider) + return MakeCloudProviderInfo(region, zone, provider) } func (self *SHost) GetShortDesc() *jsonutils.JSONDict { - info := self.getCloudBillingInfo() + info := self.getCloudProviderInfo() return jsonutils.Marshal(&info).(*jsonutils.JSONDict) } diff --git a/pkg/compute/models/managedresource.go b/pkg/compute/models/managedresource.go index 6f8fd7efd1..98412d47ac 100644 --- a/pkg/compute/models/managedresource.go +++ b/pkg/compute/models/managedresource.go @@ -1,10 +1,10 @@ package models import ( - "context" "fmt" - "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/appctx" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -30,6 +30,9 @@ func (self *SManagedResourceBase) GetCloudaccount() *SCloudaccount { func (self *SManagedResourceBase) GetDriver() (cloudprovider.ICloudProvider, error) { provider := self.GetCloudprovider() if provider == nil { + if len(self.ManagerId) > 0 { + return nil, cloudprovider.ErrInvalidProvider + } return nil, fmt.Errorf("Resource is self managed") } return provider.GetDriver() @@ -47,7 +50,7 @@ func (self *SManagedResourceBase) IsManaged() bool { return len(self.ManagerId) > 0 } -func (self *SManagedResourceBase) getExtraDetails(ctx context.Context, extra *jsonutils.JSONDict) *jsonutils.JSONDict { +/*func (self *SManagedResourceBase) getExtraDetails(ctx context.Context, extra *jsonutils.JSONDict) *jsonutils.JSONDict { manager := self.GetCloudprovider() if manager != nil { extra.Add(jsonutils.NewString(manager.Name), "manager") @@ -61,3 +64,72 @@ func (self *SManagedResourceBase) getExtraDetails(ctx context.Context, extra *js } return extra } +*/ + +type SCloudProviderInfo struct { + Provider string `json:",omitempty"` + Account string `json:",omitempty"` + AccountId string `json:",omitempty"` + Manager string `json:",omitempty"` + ManagerId string `json:",omitempty"` + ManagerProject string `json:",omitempty"` + ManagerProjectId string `json:",omitempty"` + Region string `json:",omitempty"` + RegionId string `json:",omitempty"` + RegionExtId string `json:",omitempty"` + Zone string `json:",omitempty"` + ZoneId string `json:",omitempty"` + ZoneExtId string `json:",omitempty"` +} + +func MakeCloudProviderInfo(region *SCloudregion, zone *SZone, provider *SCloudprovider) SCloudProviderInfo { + info := SCloudProviderInfo{} + + if zone != nil { + info.Zone = zone.GetName() + info.ZoneId = zone.GetId() + } + + if region != nil { + info.Region = region.GetName() + info.RegionId = region.GetId() + } + + if provider != nil { + info.Manager = provider.GetName() + info.ManagerId = provider.GetId() + + if len(provider.ProjectId) > 0 { + info.ManagerProjectId = provider.ProjectId + tc, err := db.TenantCacheManager.FetchTenantById(appctx.Background, provider.ProjectId) + if err == nil { + info.ManagerProject = tc.GetName() + } + } + + account := provider.GetCloudaccount() + info.Account = account.GetName() + info.AccountId = account.GetId() + + driver, err := provider.GetDriver() + + if err == nil { + info.Provider = driver.GetId() + + if region != nil { + iregion, err := driver.GetIRegionById(region.ExternalId) + if err == nil { + info.RegionExtId = iregion.GetId() + if zone != nil { + izone, err := iregion.GetIZoneById(zone.ExternalId) + if err == nil { + info.ZoneExtId = izone.GetId() + } + } + } + } + } + } + + return info +} diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 5e405c9c79..d436e2ecc4 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -830,7 +830,7 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND extra.Add(jsonutils.NewInt(int64(self.GetGroupNicsCount())), "group_vnics") extra.Add(jsonutils.NewInt(int64(self.GetReservedNicsCount())), "reserve_vnics") - zone := self.getZone() + /*zone := self.getZone() if zone != nil { extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") extra.Add(jsonutils.NewString(zone.GetName()), "zone") @@ -846,14 +846,14 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND if len(region.GetExternalId()) > 0 { extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id") } - } + }*/ vpc := self.getVpc() if vpc != nil { extra.Add(jsonutils.NewString(vpc.GetId()), "vpc_id") extra.Add(jsonutils.NewString(vpc.GetName()), "vpc") if len(vpc.GetExternalId()) > 0 { - extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id") + extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_ext_id") } } routes := self.GetRoutes() @@ -861,6 +861,9 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND extra.Add(jsonutils.Marshal(routes), "routes") } + info := vpc.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) + return extra } @@ -1331,6 +1334,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy. if err != nil { return nil, err } + zoneStr, _ := query.GetString("zone") if len(zoneStr) > 0 { zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zoneStr) @@ -1340,6 +1344,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy. sq := WireManager.Query("id").Equals("zone_id", zoneObj.GetId()) q = q.Filter(sqlchemy.In(q.Field("wire_id"), sq.SubQuery())) } + vpcStr, _ := query.GetString("vpc") if len(vpcStr) > 0 { vpcObj, err := VpcManager.FetchByIdOrName(userCred, vpcStr) @@ -1349,6 +1354,7 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy. sq := WireManager.Query("id").Equals("vpc_id", vpcObj.GetId()) q = q.Filter(sqlchemy.In(q.Field("wire_id"), sq.SubQuery())) } + regionStr := jsonutils.GetAnyString(query, []string{"region_id", "region", "cloudregion_id", "cloudregion"}) if len(regionStr) > 0 { region, err := CloudregionManager.FetchByIdOrName(userCred, regionStr) @@ -1367,6 +1373,63 @@ func (manager *SNetworkManager) ListItemFilter(ctx context.Context, q *sqlchemy. sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id")))) q = q.Filter(sqlchemy.In(q.Field("wire_id"), sq.SubQuery())) } + + managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"}) + if len(managerStr) > 0 { + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr) + } + return nil, httperrors.NewGeneralError(err) + } + + wires := WireManager.Query().SubQuery() + vpcs := VpcManager.Query().SubQuery() + + subq := wires.Query(wires.Field("id")) + subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) + subq = subq.Filter(sqlchemy.Equals(vpcs.Field("manager_id"), provider.GetId())) + + q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq.SubQuery())) + } + + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + + wires := WireManager.Query().SubQuery() + vpcs := VpcManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := wires.Query(wires.Field("id")) + subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId())) + + q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq.SubQuery())) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + wires := WireManager.Query().SubQuery() + vpcs := VpcManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := wires.Query(wires.Field("id")) + subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr)) + + q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq.SubQuery())) + } + return q, nil } diff --git a/pkg/compute/models/routetables.go b/pkg/compute/models/routetables.go index ebf7930615..7ef93a9665 100644 --- a/pkg/compute/models/routetables.go +++ b/pkg/compute/models/routetables.go @@ -12,6 +12,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "database/sql" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -116,6 +117,50 @@ func (man *SRouteTableManager) ListItemFilter(ctx context.Context, q *sqlchemy.S return nil, err } } + + managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"}) + if len(managerStr) > 0 { + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr) + } + return nil, httperrors.NewGeneralError(err) + } + sq := VpcManager.Query("id").Equals("manager_id", provider.GetId()) + q = q.In("vpc_id", sq.SubQuery()) + } + + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + vpcs := VpcManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := vpcs.Query(vpcs.Field("id")) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId())) + q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery())) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + vpcs := VpcManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := vpcs.Query(vpcs.Field("id")) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr)) + + q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery())) + } + return q, nil } @@ -232,6 +277,12 @@ func (rt *SRouteTable) PerformDelRoutes(ctx context.Context, userCred mcclient.T return nil, nil } +func (rt *SRouteTable) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + info := rt.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) + return extra +} + func (rt *SRouteTable) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := rt.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query) vpcM, err := VpcManager.FetchById(rt.VpcId) @@ -248,12 +299,14 @@ func (rt *SRouteTable) GetCustomizeColumns(ctx context.Context, userCred mcclien } extra.Set("vpc", jsonutils.NewString(vpcM.GetName())) extra.Set("cloudregion", jsonutils.NewString(cloudregionM.GetName())) + + extra = rt.getMoreDetails(extra) return extra } func (rt *SRouteTable) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := rt.GetCustomizeColumns(ctx, userCred, query) - extra = rt.SManagedResourceBase.getExtraDetails(ctx, extra) + extra = rt.getMoreDetails(extra) return extra } @@ -361,3 +414,26 @@ func (self *SRouteTable) SyncWithCloudRouteTable(userCred mcclient.TokenCredenti } return nil } + +func (self *SRouteTable) getVpc() (*SVpc, error) { + val, err := VpcManager.FetchById(self.VpcId) + if err != nil { + log.Errorf("VpcManager.FetchById fail %s", err) + return nil, err + } + return val.(*SVpc), nil +} + +func (self *SRouteTable) getRegion() (*SCloudregion, error) { + vpc, err := self.getVpc() + if err != nil { + return nil, err + } + return vpc.GetRegion() +} + +func (self *SRouteTable) getCloudProviderInfo() SCloudProviderInfo { + region, _ := self.getRegion() + provider := self.GetCloudprovider() + return MakeCloudProviderInfo(region, nil, provider) +} diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index ed9c81a6c4..5690b69363 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -138,6 +138,19 @@ func (manager *SSnapshotManager) ListItemFilter(ctx context.Context, q *sqlchemy q = q.Equals("manager_id", managerObj.GetId()) } + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + } + return q, nil } @@ -155,13 +168,13 @@ func (self *SSnapshot) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSON if IStorage, _ := StorageManager.FetchById(self.StorageId); IStorage != nil { storage := IStorage.(*SStorage) extra.Add(jsonutils.NewString(storage.StorageType), "storage_type") - if provider := storage.GetCloudprovider(); provider != nil { - extra.Add(jsonutils.NewString(provider.Name), "provider") - } + // if provider := storage.GetCloudprovider(); provider != nil { + // extra.Add(jsonutils.NewString(provider.Name), "provider") + // } } else { - if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { - extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") - } + // if cloudprovider := self.GetCloudprovider(); cloudprovider != nil { + // extra.Add(jsonutils.NewString(cloudprovider.Provider), "provider") + // } } disk, _ := self.GetDisk() if disk != nil { @@ -174,6 +187,10 @@ func (self *SSnapshot) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSON } extra.Add(jsonutils.NewString(disk.Name), "disk_name") } + + info := self.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) + return extra } @@ -189,9 +206,8 @@ func (self *SSnapshot) GetShortDesc() *jsonutils.JSONDict { res.Add(jsonutils.NewString(cloudRegion.ExternalId), "region") } }*/ - info := self.getCloudBillingInfo() + info := self.getCloudProviderInfo() res.Update(jsonutils.Marshal(&info)) - return res } @@ -628,8 +644,8 @@ func (self *SSnapshot) PerformPurge(ctx context.Context, userCred mcclient.Token return nil, err } -func (self *SSnapshot) getCloudBillingInfo() SCloudBillingInfo { +func (self *SSnapshot) getCloudProviderInfo() SCloudProviderInfo { region := self.GetRegion() provider := self.GetCloudprovider() - return MakeCloudBillingInfo(region, nil, provider) + return MakeCloudProviderInfo(region, nil, provider) } diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index cba7b3e590..47c26258da 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -14,6 +14,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + "database/sql" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -446,6 +447,10 @@ func (self *SStorage) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND extra.Add(jsonutils.NewFloat(0.0), "commit_rate") } extra.Add(jsonutils.NewFloat(float64(self.GetOvercommitBound())), "commit_bound") + + info := self.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) + return extra } @@ -454,6 +459,11 @@ func (self *SStorage) GetCustomizeColumns(ctx context.Context, userCred mcclient return self.getMoreDetails(extra) } +func (self *SStorage) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SStandaloneResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + func (self *SStorage) GetUsedCapacity(isReady tristate.TriState) int { disks := DiskManager.Query().SubQuery() q := disks.Query(sqlchemy.SUM("sum", disks.Field("disk_size"))).Equals("storage_id", self.Id) @@ -1112,15 +1122,37 @@ func (manager *SStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy. Filter(sqlchemy.IsTrue(q.Field("enabled"))) } - managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) + managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"}) if len(managerStr) > 0 { - provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr) - if provider == nil { - return nil, httperrors.NewResourceNotFoundError("provider %s not found", managerStr) + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr) + } + return nil, httperrors.NewGeneralError(err) } q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId())) } + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + } + return q, err } @@ -1141,17 +1173,17 @@ func (self *SStorage) ClearSchedDescCache() error { return nil } -func (self *SStorage) getCloudBillingInfo() SCloudBillingInfo { +func (self *SStorage) getCloudProviderInfo() SCloudProviderInfo { var region *SCloudregion zone := self.getZone() if zone != nil { region = zone.GetRegion() } provider := self.GetCloudprovider() - return MakeCloudBillingInfo(region, zone, provider) + return MakeCloudProviderInfo(region, zone, provider) } func (self *SStorage) GetShortDesc() *jsonutils.JSONDict { - info := self.getCloudBillingInfo() + info := self.getCloudProviderInfo() return jsonutils.Marshal(&info).(*jsonutils.JSONDict) } diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index 2d1b3bc402..55b3af55a3 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -102,7 +102,7 @@ func (self *SVpc) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCr } func (self *SVpc) ValidateDeleteCondition(ctx context.Context) error { - if self.GetNetworkCount() > 0 { + if self.GetNetworkCount() > 0 || self.GetRouteTableCount() > 0 { return httperrors.NewNotEmptyError("VPC not empty") } if self.Id == DEFAULT_VPC_ID { @@ -150,10 +150,15 @@ func (self *SVpc) GetNetworkCount() int { return q.Count() } +func (self *SVpc) GetRouteTableCount() int { + return RouteTableManager.Query().Equals("vpc_id", self.Id).Count() +} + func (self *SVpc) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { extra.Add(jsonutils.NewInt(int64(self.GetWireCount())), "wire_count") extra.Add(jsonutils.NewInt(int64(self.GetNetworkCount())), "network_count") - region, err := self.GetRegion() + extra.Add(jsonutils.NewInt(int64(self.GetRouteTableCount())), "routetable_count") + /* region, err := self.GetRegion() if err != nil { log.Errorf("failed getting region for vpc %s(%s)", self.Name, self.Id) return extra @@ -161,10 +166,20 @@ func (self *SVpc) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict extra.Add(jsonutils.NewString(region.GetName()), "region") if len(region.GetExternalId()) > 0 { extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id") - } + }*/ + + info := self.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) + return extra } +func (self *SVpc) getCloudProviderInfo() SCloudProviderInfo { + region, _ := self.GetRegion() + provider := self.GetCloudprovider() + return MakeCloudProviderInfo(region, nil, provider) +} + func (self *SVpc) GetRegion() (*SCloudregion, error) { region, err := CloudregionManager.FetchById(self.CloudregionId) if err != nil { @@ -510,18 +525,43 @@ func (self *SVpc) PerformPurge(ctx context.Context, userCred mcclient.TokenCrede } func (manager *SVpcManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + queryDict := query.(*jsonutils.JSONDict) + + managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"}) + if len(managerStr) > 0 { + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr) + } + return nil, httperrors.NewGeneralError(err) + } + q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId())) + queryDict.Remove("manager_id") + } + q, err := manager.SStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query) if err != nil { return nil, err } - managerStr := jsonutils.GetAnyString(query, []string{"manager", "provider", "manager_id", "provider_id"}) - if len(managerStr) > 0 { - provider := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr) - if provider == nil { - return nil, httperrors.NewResourceNotFoundError("provider %s not found", managerStr) + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) } - q = q.Filter(sqlchemy.Equals(q.Field("manager_id"), provider.GetId())) + subq := CloudproviderManager.Query("id").Equals("cloudaccount_id", account.GetId()).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery() + q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq)) } return q, nil diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 7b9359d530..3bfcef87a5 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -632,6 +632,49 @@ func (manager *SWireManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu q = q.In("vpc_id", sq.SubQuery()) } + managerStr := jsonutils.GetAnyString(query, []string{"manager", "cloudprovider", "cloudprovider_id", "manager_id"}) + if len(managerStr) > 0 { + provider, err := CloudproviderManager.FetchByIdOrName(nil, managerStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr) + } + return nil, httperrors.NewGeneralError(err) + } + sq := VpcManager.Query("id").Equals("manager_id", provider.GetId()) + q = q.In("vpc_id", sq.SubQuery()) + } + + accountStr := jsonutils.GetAnyString(query, []string{"account", "account_id", "cloudaccount", "cloudaccount_id"}) + if len(accountStr) > 0 { + account, err := CloudaccountManager.FetchByIdOrName(nil, accountStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr) + } + return nil, httperrors.NewGeneralError(err) + } + vpcs := VpcManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := vpcs.Query(vpcs.Field("id")) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("cloudaccount_id"), account.GetId())) + q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery())) + } + + providerStr := jsonutils.GetAnyString(query, []string{"provider"}) + if len(providerStr) > 0 { + vpcs := VpcManager.Query().SubQuery() + cloudproviders := CloudproviderManager.Query().SubQuery() + + subq := vpcs.Query(vpcs.Field("id")) + subq = subq.Join(cloudproviders, sqlchemy.Equals(cloudproviders.Field("id"), vpcs.Field("manager_id"))) + subq = subq.Filter(sqlchemy.Equals(cloudproviders.Field("provider"), providerStr)) + + q = q.Filter(sqlchemy.In(q.Field("vpc_id"), subq.SubQuery())) + } + return q, err } @@ -655,7 +698,7 @@ func (self *SWire) GetExtraDetails(ctx context.Context, userCred mcclient.TokenC func (self *SWire) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { extra.Add(jsonutils.NewInt(int64(self.NetworkCount())), "networks") - zone := self.GetZone() + /*zone := self.GetZone() if zone != nil { extra.Add(jsonutils.NewString(zone.GetName()), "zone") if len(zone.GetExternalId()) > 0 { @@ -669,13 +712,17 @@ func (self *SWire) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict if len(region.GetExternalId()) > 0 { extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id") } - } + }*/ vpc := self.getVpc() if vpc != nil { extra.Add(jsonutils.NewString(vpc.GetName()), "vpc") if len(vpc.GetExternalId()) > 0 { - extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id") + extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_ext_id") } } + + info := vpc.getCloudProviderInfo() + extra.Update(jsonutils.Marshal(&info)) + return extra } diff --git a/pkg/compute/tasks/eip_deallocate_task.go b/pkg/compute/tasks/eip_deallocate_task.go index b483e7a5b1..15d2b53938 100644 --- a/pkg/compute/tasks/eip_deallocate_task.go +++ b/pkg/compute/tasks/eip_deallocate_task.go @@ -27,7 +27,7 @@ func (self *EipDeallocateTask) OnInit(ctx context.Context, obj db.IStandaloneMod if len(eip.ExternalId) > 0 { expEip, err := eip.GetIEip() if err != nil { - if err != cloudprovider.ErrNotFound { + if err != cloudprovider.ErrNotFound && err != cloudprovider.ErrInvalidProvider { msg := fmt.Sprintf("fail to find iEIP for eip %s", err) eip.SetStatus(self.UserCred, models.EIP_STATUS_DEALLOCATE_FAIL, msg) self.SetStageFailed(ctx, msg) diff --git a/pkg/mcclient/options/routetables.go b/pkg/mcclient/options/routetables.go index c214d824b8..374eb0f8a4 100644 --- a/pkg/mcclient/options/routetables.go +++ b/pkg/mcclient/options/routetables.go @@ -129,5 +129,9 @@ type RouteTableListOptions struct { Vpc string Cloudregion string + Manager string `help:"List hosts belongs to the cloud provider"` + Account string `help:"List hosts belongs to the cloud account"` + Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` + BaseListOptions } diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index db3727a3ee..4f1d56fe34 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -19,11 +19,14 @@ type ServerListOptions struct { Secgroup string `help:"Secgroup ID or Name"` AdminSecgroup string `help:"AdminSecgroup ID or Name"` Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws"` - Manager string `help:"Show servers imported from manager"` Region string `help:"Show servers in cloudregion"` WithEip *bool `help:"Show Servers with EIP"` WithoutEip *bool `help:"Show Servers without EIP"` + Manager string `help:"Show servers imported from manager"` + Account string `help:"List hosts belongs to the cloud account"` + Provider string `help:"List hosts belongs to the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei"` + ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` BillingType string `help:"billing type" choices:"postpaid|prepaid"` From 56370457ddc4360d7db19311736b55e5f0478ca0 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 12 Dec 2018 14:08:14 +0000 Subject: [PATCH 32/34] =?UTF-8?q?climc:=20=E5=9C=A8stderr=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E9=94=99=E8=AF=AF=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/climc/climc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index 175543ccc8..df805d5238 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -84,7 +84,7 @@ func getSubcommandsParser() (*structarg.ArgumentParser, error) { } func showErrorAndExit(e error) { - fmt.Printf("Error: %s\n", e) + fmt.Fprintf(os.Stderr, "Error: %s\n", e) os.Exit(1) } From 46aaba247414d502bf38764af7347966e0c51b46 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 23:24:34 +0800 Subject: [PATCH 33/34] make fmt --- pkg/compute/guestdrivers/aliyun.go | 4 ++-- pkg/compute/guestdrivers/aws.go | 4 ++-- pkg/compute/guestdrivers/azure.go | 12 ++++++------ pkg/compute/guestdrivers/base.go | 2 +- pkg/compute/guestdrivers/esxi.go | 2 +- pkg/compute/guestdrivers/managedvirtual.go | 4 ++-- pkg/compute/guestdrivers/qcloud.go | 4 ++-- pkg/compute/guestdrivers/virtualization.go | 4 ++-- pkg/compute/hostdrivers/base.go | 3 ++- pkg/compute/hostdrivers/esxi.go | 1 + pkg/compute/hostdrivers/kvm.go | 1 + pkg/compute/hostdrivers/managedvirtual.go | 8 +++----- pkg/compute/misc/handler.go | 3 ++- pkg/compute/models/routetables.go | 2 +- pkg/compute/models/schedpolicies.go | 6 +++--- pkg/compute/models/schedtags.go | 5 +++-- pkg/compute/models/secgroupcache.go | 5 +++-- pkg/compute/models/secgrouprules.go | 7 ++++--- pkg/compute/models/skus.go | 13 ++++++------- pkg/compute/models/storagecachedimages.go | 1 + pkg/compute/models/storagecaches.go | 1 + pkg/compute/models/storages.go | 2 +- pkg/compute/models/usage.go | 2 +- pkg/compute/models/vcenters.go | 1 + pkg/compute/models/wires.go | 9 +++++---- pkg/compute/models/zones.go | 7 ++++--- pkg/util/aliyun/aliyun.go | 3 ++- pkg/util/aliyun/disk.go | 5 +++-- pkg/util/aliyun/eip.go | 3 +-- pkg/util/aliyun/host.go | 1 + pkg/util/aliyun/image.go | 4 +++- pkg/util/aliyun/instance.go | 5 ++--- pkg/util/aliyun/keypair.go | 1 + pkg/util/aliyun/ram.go | 1 + pkg/util/aliyun/snapshot.go | 1 + pkg/util/aliyun/storage.go | 1 + pkg/util/aliyun/storagecache.go | 1 + pkg/util/aliyun/vpc.go | 1 + pkg/util/aliyun/vswitch.go | 5 +++-- pkg/util/aliyun/wire.go | 1 + pkg/util/aliyun/zone.go | 1 + pkg/util/aws/aws.go | 9 +++++---- pkg/util/aws/disk.go | 8 +++++--- pkg/util/aws/eip.go | 2 ++ pkg/util/aws/host.go | 2 ++ pkg/util/aws/image.go | 4 +++- pkg/util/aws/instance.go | 7 ++++--- pkg/util/aws/instancenic.go | 3 ++- pkg/util/aws/instancetype.go | 2 ++ pkg/util/aws/keypair.go | 6 ++++-- pkg/util/aws/network.go | 7 +++++-- pkg/util/aws/region.go | 2 ++ pkg/util/aws/securitygroup.go | 4 +++- pkg/util/aws/snapshot.go | 5 ++++- pkg/util/aws/storage.go | 2 ++ pkg/util/aws/storagecache.go | 11 ++++++----- pkg/util/aws/utils.go | 3 ++- pkg/util/aws/vpc.go | 5 +++-- pkg/util/aws/wire.go | 2 ++ pkg/util/aws/zone.go | 2 ++ pkg/util/azure/azure.go | 1 + pkg/util/azure/classic_disk.go | 3 ++- pkg/util/azure/classic_eip.go | 1 + pkg/util/azure/classic_host.go | 1 + pkg/util/azure/classic_instance.go | 6 +++--- pkg/util/azure/classic_instancenic.go | 3 ++- pkg/util/azure/classic_network.go | 3 ++- pkg/util/azure/classic_secruitygroup.go | 3 ++- pkg/util/azure/classic_snapshot.go | 1 + pkg/util/azure/classic_storage.go | 1 + pkg/util/azure/classic_vpc.go | 1 + pkg/util/azure/classic_wire.go | 1 + pkg/util/azure/debug.go | 4 ++-- pkg/util/azure/disk.go | 3 ++- pkg/util/azure/eip.go | 1 + pkg/util/azure/host.go | 1 + pkg/util/azure/image.go | 3 ++- pkg/util/azure/instance.go | 6 +++--- pkg/util/azure/instancenic.go | 3 ++- pkg/util/azure/network.go | 3 ++- pkg/util/azure/region.go | 3 ++- pkg/util/azure/snapshot.go | 1 + pkg/util/azure/storage.go | 1 + pkg/util/azure/storageaccount.go | 2 ++ pkg/util/azure/storagecache.go | 1 + pkg/util/azure/upload.go | 6 +++--- pkg/util/azure/vpc.go | 1 + pkg/util/azure/wire.go | 1 + pkg/util/azure/zone.go | 1 + pkg/util/billing/billingcycle.go | 1 + pkg/util/cloudinit/cloudconfig.go | 5 +++-- pkg/util/conditionparser/parser.go | 3 +-- pkg/util/esxi/datacenter.go | 2 ++ pkg/util/esxi/device.go | 1 - pkg/util/esxi/image.go | 3 ++- pkg/util/esxi/storage.go | 10 ++++++---- pkg/util/esxi/virtualmachine.go | 7 +++---- pkg/util/esxi/vnic.go | 3 ++- pkg/util/qcloud/disk.go | 5 +++-- pkg/util/qcloud/eip.go | 1 + pkg/util/qcloud/host.go | 1 + pkg/util/qcloud/image.go | 5 +++-- pkg/util/qcloud/instance.go | 3 ++- pkg/util/qcloud/instancenic.go | 3 ++- pkg/util/qcloud/keypair.go | 2 ++ pkg/util/qcloud/localdisk.go | 3 ++- pkg/util/qcloud/localstorage.go | 1 + pkg/util/qcloud/network.go | 3 ++- pkg/util/qcloud/qcloud.go | 2 ++ pkg/util/qcloud/region.go | 4 +++- pkg/util/qcloud/securitygroup.go | 3 ++- pkg/util/qcloud/snapshot.go | 1 + pkg/util/qcloud/storage.go | 1 + pkg/util/qcloud/storagecache.go | 5 +++-- pkg/util/qcloud/vpc.go | 1 + pkg/util/qcloud/wire.go | 1 + pkg/util/qcloud/zone.go | 3 ++- pkg/util/rbacutils/rabc.go | 1 + pkg/util/vmdkutils/vmdkutils.go | 4 ++-- 119 files changed, 248 insertions(+), 139 deletions(-) diff --git a/pkg/compute/guestdrivers/aliyun.go b/pkg/compute/guestdrivers/aliyun.go index bb68155d0e..e624b61efb 100644 --- a/pkg/compute/guestdrivers/aliyun.go +++ b/pkg/compute/guestdrivers/aliyun.go @@ -7,13 +7,13 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/seclib2" ) diff --git a/pkg/compute/guestdrivers/aws.go b/pkg/compute/guestdrivers/aws.go index ee0dc499db..1411df2c5c 100644 --- a/pkg/compute/guestdrivers/aws.go +++ b/pkg/compute/guestdrivers/aws.go @@ -5,14 +5,14 @@ import ( "fmt" "time" - "yunion.io/x/onecloud/pkg/util/ansible" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/ansible" "yunion.io/x/onecloud/pkg/util/billing" ) diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index ff2643afb2..6004d216d5 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -6,20 +6,20 @@ import ( "strings" "time" + "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/ansible" - "yunion.io/x/onecloud/pkg/util/seclib2" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" - "yunion.io/x/jsonutils" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/ansible" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/seclib2" ) type SAzureGuestDriver struct { diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 572345da30..2c62d65481 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -4,11 +4,11 @@ import ( "context" "fmt" "net/http" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" - "time" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" diff --git a/pkg/compute/guestdrivers/esxi.go b/pkg/compute/guestdrivers/esxi.go index b9064c5946..68fbbc9a5a 100644 --- a/pkg/compute/guestdrivers/esxi.go +++ b/pkg/compute/guestdrivers/esxi.go @@ -4,12 +4,12 @@ import ( "context" "fmt" "net/http" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/utils" - "time" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 7e233158f4..6082eff953 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -3,19 +3,19 @@ package guestdrivers import ( "context" "fmt" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/secrules" - "time" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" ) diff --git a/pkg/compute/guestdrivers/qcloud.go b/pkg/compute/guestdrivers/qcloud.go index cd831f4adb..5090af0b30 100644 --- a/pkg/compute/guestdrivers/qcloud.go +++ b/pkg/compute/guestdrivers/qcloud.go @@ -7,8 +7,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/utils" @@ -16,6 +14,8 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/seclib2" ) diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index a243caf7bf..11fb8ab16c 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -7,12 +7,12 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SVirtualizedGuestDriver struct { diff --git a/pkg/compute/hostdrivers/base.go b/pkg/compute/hostdrivers/base.go index 6dcaea62f9..9adc40c7bc 100644 --- a/pkg/compute/hostdrivers/base.go +++ b/pkg/compute/hostdrivers/base.go @@ -6,11 +6,12 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/utils" ) type SBaseHostDriver struct { diff --git a/pkg/compute/hostdrivers/esxi.go b/pkg/compute/hostdrivers/esxi.go index d7d1158007..4b7ef28661 100644 --- a/pkg/compute/hostdrivers/esxi.go +++ b/pkg/compute/hostdrivers/esxi.go @@ -3,6 +3,7 @@ package hostdrivers import ( "context" "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 83e33fc4eb..bb050254b2 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/compute/models" diff --git a/pkg/compute/hostdrivers/managedvirtual.go b/pkg/compute/hostdrivers/managedvirtual.go index b54bef5660..f1179e1bd2 100644 --- a/pkg/compute/hostdrivers/managedvirtual.go +++ b/pkg/compute/hostdrivers/managedvirtual.go @@ -8,14 +8,12 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - "yunion.io/x/onecloud/pkg/compute/options" - + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/compute/options" + "yunion.io/x/onecloud/pkg/httperrors" ) type SManagedVirtualizationHostDriver struct { diff --git a/pkg/compute/misc/handler.go b/pkg/compute/misc/handler.go index 5c9ea1cd98..7e3c485785 100644 --- a/pkg/compute/misc/handler.go +++ b/pkg/compute/misc/handler.go @@ -6,13 +6,14 @@ import ( "net/http" "yunion.io/x/log" + "yunion.io/x/pkg/tristate" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/pkg/tristate" ) func AddMiscHandler(prefix string, app *appsrv.Application) { diff --git a/pkg/compute/models/routetables.go b/pkg/compute/models/routetables.go index b7708a110a..7c586f6e19 100644 --- a/pkg/compute/models/routetables.go +++ b/pkg/compute/models/routetables.go @@ -2,6 +2,7 @@ package models import ( "context" + "database/sql" "net" "reflect" "strings" @@ -12,7 +13,6 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" - "database/sql" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" "yunion.io/x/onecloud/pkg/cloudprovider" diff --git a/pkg/compute/models/schedpolicies.go b/pkg/compute/models/schedpolicies.go index e4a54dafef..745a9d6558 100644 --- a/pkg/compute/models/schedpolicies.go +++ b/pkg/compute/models/schedpolicies.go @@ -2,16 +2,16 @@ package models import ( "context" + "database/sql" "yunion.io/x/jsonutils" - - "database/sql" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/conditionparser" - "yunion.io/x/pkg/utils" ) type SSchedpolicyManager struct { diff --git a/pkg/compute/models/schedtags.go b/pkg/compute/models/schedtags.go index b4605d7e9a..c8c7a73083 100644 --- a/pkg/compute/models/schedtags.go +++ b/pkg/compute/models/schedtags.go @@ -7,11 +7,12 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/utils" - "yunion.io/x/sqlchemy" ) type SchedStrategyType string diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index ca52a2d245..db741f973c 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -6,13 +6,14 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/stringutils" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/util/stringutils" - "yunion.io/x/sqlchemy" ) type SSecurityGroupCacheManager struct { diff --git a/pkg/compute/models/secgrouprules.go b/pkg/compute/models/secgrouprules.go index 4c0c1ae6f9..8745d4dde3 100644 --- a/pkg/compute/models/secgrouprules.go +++ b/pkg/compute/models/secgrouprules.go @@ -9,14 +9,15 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SSecurityGroupRuleManager struct { diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index 88fc8df1d2..f3ef8529ec 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -3,7 +3,6 @@ package models import ( "context" "database/sql" - "encoding/json" "fmt" "yunion.io/x/jsonutils" @@ -26,7 +25,7 @@ const ( SkuCategoryHighMemory = "high_memory" // 高内存型 ) -var InstanceFamilies map[string]string = map[string]string{ +var InstanceFamilies = map[string]string{ SkuCategoryGeneralPurpose: "g1", SkuCategoryBurstable: "t1", SkuCategoryComputeOptimized: "c1", @@ -312,17 +311,17 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use ret.Add(cpus, "cpus") ret.Add(mems_mb, "mems_mb") - r, err := json.Marshal(&cpu_mems_mb) + /* r, err := json.Marshal(&cpu_mems_mb) if err != nil { log.Errorf("%s", err) return nil, httperrors.NewInternalServerError("instance specs list marshal failed") - } + }*/ - r_obj, err := jsonutils.Parse(r) - if err != nil { + r_obj := jsonutils.Marshal(&cpu_mems_mb) + /*if err != nil { log.Errorf("%s", err) return nil, httperrors.NewInternalServerError("instance specs list parse failed") - } + }*/ ret.Add(r_obj, "cpu_mems_mb") return ret, nil diff --git a/pkg/compute/models/storagecachedimages.go b/pkg/compute/models/storagecachedimages.go index a2aedb8b51..7cf5d24532 100644 --- a/pkg/compute/models/storagecachedimages.go +++ b/pkg/compute/models/storagecachedimages.go @@ -6,6 +6,7 @@ import ( "time" "github.com/serialx/hashring" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/utils" diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index b4ee7d0447..5b9abed6ad 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/serialx/hashring" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/sqlchemy" diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 47c26258da..709dcda1ce 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -2,6 +2,7 @@ package models import ( "context" + "database/sql" "fmt" "path" "strings" @@ -14,7 +15,6 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" - "database/sql" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" diff --git a/pkg/compute/models/usage.go b/pkg/compute/models/usage.go index b933b03772..ab36e2212d 100644 --- a/pkg/compute/models/usage.go +++ b/pkg/compute/models/usage.go @@ -1,10 +1,10 @@ package models import ( + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/pkg/utils" ) func AttachUsageQuery( diff --git a/pkg/compute/models/vcenters.go b/pkg/compute/models/vcenters.go index f4ebc43687..cc8f428fd1 100644 --- a/pkg/compute/models/vcenters.go +++ b/pkg/compute/models/vcenters.go @@ -5,6 +5,7 @@ import ( "time" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/mcclient" ) diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 3bfcef87a5..7620e3b283 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -7,14 +7,15 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/netutils" "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" ) type SWireManager struct { diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index 19a7ec5974..b327e6c50a 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -6,13 +6,14 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/util/compare" + "yunion.io/x/sqlchemy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/pkg/tristate" - "yunion.io/x/pkg/util/compare" - "yunion.io/x/sqlchemy" ) const ( diff --git a/pkg/util/aliyun/aliyun.go b/pkg/util/aliyun/aliyun.go index bb8e66ef9c..756fd89227 100644 --- a/pkg/util/aliyun/aliyun.go +++ b/pkg/util/aliyun/aliyun.go @@ -8,9 +8,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/utils" ) const ( diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index 4a58d6d94e..e5bd5c23c5 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -1,15 +1,16 @@ package aliyun import ( + "context" "fmt" "time" - "context" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/utils" ) type SMountInstances struct { diff --git a/pkg/util/aliyun/eip.go b/pkg/util/aliyun/eip.go index 54534bfa82..7c77a4f1b3 100644 --- a/pkg/util/aliyun/eip.go +++ b/pkg/util/aliyun/eip.go @@ -4,9 +4,8 @@ import ( "fmt" "time" - "yunion.io/x/log" - "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudprovider" diff --git a/pkg/util/aliyun/host.go b/pkg/util/aliyun/host.go index 5c4e545ffb..0702b96e77 100644 --- a/pkg/util/aliyun/host.go +++ b/pkg/util/aliyun/host.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" diff --git a/pkg/util/aliyun/image.go b/pkg/util/aliyun/image.go index a34713576f..44b270ff5a 100644 --- a/pkg/util/aliyun/image.go +++ b/pkg/util/aliyun/image.go @@ -1,14 +1,16 @@ package aliyun import ( + "context" "fmt" "strings" "time" - "context" "github.com/aliyun/aliyun-oss-go-sdk/oss" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 20717d49db..a48a607049 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -1,7 +1,9 @@ package aliyun import ( + "context" "fmt" + "sort" "time" "yunion.io/x/jsonutils" @@ -10,9 +12,6 @@ import ( "yunion.io/x/pkg/util/seclib" "yunion.io/x/pkg/utils" - "context" - - "sort" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" diff --git a/pkg/util/aliyun/keypair.go b/pkg/util/aliyun/keypair.go index deead623db..5be8d34d1b 100644 --- a/pkg/util/aliyun/keypair.go +++ b/pkg/util/aliyun/keypair.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "time" + "yunion.io/x/log" ) diff --git a/pkg/util/aliyun/ram.go b/pkg/util/aliyun/ram.go index 8e0d2814fb..a9290d494d 100644 --- a/pkg/util/aliyun/ram.go +++ b/pkg/util/aliyun/ram.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/aliyun/snapshot.go b/pkg/util/aliyun/snapshot.go index e087e3a673..bfbc5b21c5 100644 --- a/pkg/util/aliyun/snapshot.go +++ b/pkg/util/aliyun/snapshot.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aliyun/storage.go b/pkg/util/aliyun/storage.go index 26c1a729c2..53a5791f09 100644 --- a/pkg/util/aliyun/storage.go +++ b/pkg/util/aliyun/storage.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go index 54c84623dd..49dbe73a3c 100644 --- a/pkg/util/aliyun/storagecache.go +++ b/pkg/util/aliyun/storagecache.go @@ -10,6 +10,7 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" compute "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/compute/options" diff --git a/pkg/util/aliyun/vpc.go b/pkg/util/aliyun/vpc.go index 402156dcf1..bc29b5d5c7 100644 --- a/pkg/util/aliyun/vpc.go +++ b/pkg/util/aliyun/vpc.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/aliyun/vswitch.go b/pkg/util/aliyun/vswitch.go index 9b93a16603..944f9323b0 100644 --- a/pkg/util/aliyun/vswitch.go +++ b/pkg/util/aliyun/vswitch.go @@ -6,10 +6,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" ) // {"AvailableIpAddressCount":4091,"CidrBlock":"172.31.32.0/20","CreationTime":"2017-03-19T13:37:44Z","Description":"System created default virtual switch.","IsDefault":true,"Status":"Available","VSwitchId":"vsw-j6c3gig5ub4fmi2veyrus","VSwitchName":"","VpcId":"vpc-j6c86z3sh8ufhgsxwme0q","ZoneId":"cn-hongkong-b"} diff --git a/pkg/util/aliyun/wire.go b/pkg/util/aliyun/wire.go index 12793359c1..0c3c5de0e9 100644 --- a/pkg/util/aliyun/wire.go +++ b/pkg/util/aliyun/wire.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/aliyun/zone.go b/pkg/util/aliyun/zone.go index 2241e4d9ce..bba20ddd28 100644 --- a/pkg/util/aliyun/zone.go +++ b/pkg/util/aliyun/zone.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aws/aws.go b/pkg/util/aws/aws.go index bbd19630e1..82866600e8 100644 --- a/pkg/util/aws/aws.go +++ b/pkg/util/aws/aws.go @@ -1,14 +1,15 @@ package aws import ( - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/models" - sdk "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" + + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" ) const ( diff --git a/pkg/util/aws/disk.go b/pkg/util/aws/disk.go index 8070068e7a..703586bbaa 100644 --- a/pkg/util/aws/disk.go +++ b/pkg/util/aws/disk.go @@ -1,15 +1,17 @@ package aws import ( + "context" "fmt" "sort" "strings" "time" - "context" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/coredns/coredns/plugin/pkg/log" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) @@ -296,7 +298,7 @@ func (self *SRegion) GetDisks(instanceId string, zoneId string, storageType stri if len(disk.InstanceId) > 0 { instance, err := self.GetInstance(disk.InstanceId) if err != nil { - log.Debug(err) + log.Debugf("%s", err) return nil, 0, err } diff --git a/pkg/util/aws/eip.go b/pkg/util/aws/eip.go index 61fac8ac4f..c259c8f2ae 100644 --- a/pkg/util/aws/eip.go +++ b/pkg/util/aws/eip.go @@ -5,8 +5,10 @@ import ( "time" "github.com/aws/aws-sdk-go/service/ec2" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aws/host.go b/pkg/util/aws/host.go index 07552c1d0e..2a8578b4a3 100644 --- a/pkg/util/aws/host.go +++ b/pkg/util/aws/host.go @@ -2,8 +2,10 @@ package aws import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" diff --git a/pkg/util/aws/image.go b/pkg/util/aws/image.go index 2df0ae1f4b..370bf11e78 100644 --- a/pkg/util/aws/image.go +++ b/pkg/util/aws/image.go @@ -1,13 +1,15 @@ package aws import ( + "context" "fmt" "strings" - "context" "github.com/aws/aws-sdk-go/service/ec2" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aws/instance.go b/pkg/util/aws/instance.go index 963798331d..0bf3db1e06 100644 --- a/pkg/util/aws/instance.go +++ b/pkg/util/aws/instance.go @@ -1,19 +1,20 @@ package aws import ( + "context" "fmt" "strings" "time" - "context" - "github.com/aws/aws-sdk-go/service/ec2" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" - "yunion.io/x/pkg/util/osprofile" ) const ( diff --git a/pkg/util/aws/instancenic.go b/pkg/util/aws/instancenic.go index 24b64b2924..1b5814b948 100644 --- a/pkg/util/aws/instancenic.go +++ b/pkg/util/aws/instancenic.go @@ -1,8 +1,9 @@ package aws import ( - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SInstanceNic struct { diff --git a/pkg/util/aws/instancetype.go b/pkg/util/aws/instancetype.go index 796a3fc84d..1ac44719dc 100644 --- a/pkg/util/aws/instancetype.go +++ b/pkg/util/aws/instancetype.go @@ -4,7 +4,9 @@ import ( "encoding/json" "fmt" "io/ioutil" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/compute/options" ) diff --git a/pkg/util/aws/keypair.go b/pkg/util/aws/keypair.go index 3bbf8a7104..762892aa2b 100644 --- a/pkg/util/aws/keypair.go +++ b/pkg/util/aws/keypair.go @@ -2,11 +2,13 @@ package aws import ( "fmt" + "strconv" + "time" + "github.com/aokoli/goutils" "github.com/aws/aws-sdk-go/service/ec2" "golang.org/x/crypto/ssh" - "strconv" - "time" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/aws/network.go b/pkg/util/aws/network.go index 3af8d48ef3..4e6f26dacf 100644 --- a/pkg/util/aws/network.go +++ b/pkg/util/aws/network.go @@ -1,14 +1,17 @@ package aws import ( - "github.com/aws/aws-sdk-go/service/ec2" "strings" "time" + + "github.com/aws/aws-sdk-go/service/ec2" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/util/netutils" ) type SNetwork struct { diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go index d7ee3eac23..4103613793 100644 --- a/pkg/util/aws/region.go +++ b/pkg/util/aws/region.go @@ -9,8 +9,10 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aws/securitygroup.go b/pkg/util/aws/securitygroup.go index 1d06c72dce..ded277dfdf 100644 --- a/pkg/util/aws/securitygroup.go +++ b/pkg/util/aws/securitygroup.go @@ -7,10 +7,12 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang-plus/uuid" + "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/pkg/util/secrules" + + "yunion.io/x/onecloud/pkg/httperrors" ) type Tags struct { diff --git a/pkg/util/aws/snapshot.go b/pkg/util/aws/snapshot.go index eb61b9c38a..7b0b775fe3 100644 --- a/pkg/util/aws/snapshot.go +++ b/pkg/util/aws/snapshot.go @@ -2,10 +2,13 @@ package aws import ( "fmt" - "github.com/aws/aws-sdk-go/service/ec2" "strings" + + "github.com/aws/aws-sdk-go/service/ec2" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aws/storage.go b/pkg/util/aws/storage.go index 937598edcc..4da587ec70 100644 --- a/pkg/util/aws/storage.go +++ b/pkg/util/aws/storage.go @@ -2,8 +2,10 @@ package aws import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/aws/storagecache.go b/pkg/util/aws/storagecache.go index 9d0da0cba8..071004d619 100644 --- a/pkg/util/aws/storagecache.go +++ b/pkg/util/aws/storagecache.go @@ -5,18 +5,19 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" - - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" ) type SStoragecache struct { diff --git a/pkg/util/aws/utils.go b/pkg/util/aws/utils.go index ca10292a68..15c1090599 100644 --- a/pkg/util/aws/utils.go +++ b/pkg/util/aws/utils.go @@ -6,9 +6,10 @@ import ( "reflect" "regexp" "strings" - "yunion.io/x/jsonutils" "github.com/aws/aws-sdk-go/service/ec2" + + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/util/secrules" ) diff --git a/pkg/util/aws/vpc.go b/pkg/util/aws/vpc.go index cd9543a8c9..75312d1545 100644 --- a/pkg/util/aws/vpc.go +++ b/pkg/util/aws/vpc.go @@ -4,12 +4,13 @@ import ( "fmt" "strings" + "github.com/aws/aws-sdk-go/service/ec2" + "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/secrules" - "github.com/aws/aws-sdk-go/service/ec2" + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SUserCIDRs struct { diff --git a/pkg/util/aws/wire.go b/pkg/util/aws/wire.go index 9863117552..fe62af3b02 100644 --- a/pkg/util/aws/wire.go +++ b/pkg/util/aws/wire.go @@ -2,8 +2,10 @@ package aws import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/aws/zone.go b/pkg/util/aws/zone.go index eecccc70ac..7b147af36c 100644 --- a/pkg/util/aws/zone.go +++ b/pkg/util/aws/zone.go @@ -2,8 +2,10 @@ package aws import ( "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/azure.go b/pkg/util/azure/azure.go index 634aac1b9b..ca05703077 100644 --- a/pkg/util/azure/azure.go +++ b/pkg/util/azure/azure.go @@ -14,6 +14,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" diff --git a/pkg/util/azure/classic_disk.go b/pkg/util/azure/classic_disk.go index 58a09808ac..922e7ab84d 100644 --- a/pkg/util/azure/classic_disk.go +++ b/pkg/util/azure/classic_disk.go @@ -1,12 +1,13 @@ package azure import ( + "context" "strings" "time" - "context" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/classic_eip.go b/pkg/util/azure/classic_eip.go index 844ed917a8..901dd900b6 100644 --- a/pkg/util/azure/classic_eip.go +++ b/pkg/util/azure/classic_eip.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/classic_host.go b/pkg/util/azure/classic_host.go index 88ec9f992d..85988cc207 100644 --- a/pkg/util/azure/classic_host.go +++ b/pkg/util/azure/classic_host.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" diff --git a/pkg/util/azure/classic_instance.go b/pkg/util/azure/classic_instance.go index a7b6dbc391..4f7add3332 100644 --- a/pkg/util/azure/classic_instance.go +++ b/pkg/util/azure/classic_instance.go @@ -1,18 +1,18 @@ package azure import ( + "context" "fmt" "strings" "time" - "context" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" - "yunion.io/x/pkg/util/osprofile" ) type FormattedMessage struct { diff --git a/pkg/util/azure/classic_instancenic.go b/pkg/util/azure/classic_instancenic.go index 0b32999ac3..cbdab1a65a 100644 --- a/pkg/util/azure/classic_instancenic.go +++ b/pkg/util/azure/classic_instancenic.go @@ -2,8 +2,9 @@ package azure import ( "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SClassicInstanceNic struct { diff --git a/pkg/util/azure/classic_network.go b/pkg/util/azure/classic_network.go index 6f328299f3..c266bf172b 100644 --- a/pkg/util/azure/classic_network.go +++ b/pkg/util/azure/classic_network.go @@ -4,9 +4,10 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/util/netutils" ) type SClassicNetwork struct { diff --git a/pkg/util/azure/classic_secruitygroup.go b/pkg/util/azure/classic_secruitygroup.go index 54baefd144..d472924704 100644 --- a/pkg/util/azure/classic_secruitygroup.go +++ b/pkg/util/azure/classic_secruitygroup.go @@ -9,9 +9,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SClassicSecurityGroup struct { diff --git a/pkg/util/azure/classic_snapshot.go b/pkg/util/azure/classic_snapshot.go index e5cb1d74b0..561d515538 100644 --- a/pkg/util/azure/classic_snapshot.go +++ b/pkg/util/azure/classic_snapshot.go @@ -4,6 +4,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/classic_storage.go b/pkg/util/azure/classic_storage.go index 329f965637..d1cf9acbcc 100644 --- a/pkg/util/azure/classic_storage.go +++ b/pkg/util/azure/classic_storage.go @@ -4,6 +4,7 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/classic_vpc.go b/pkg/util/azure/classic_vpc.go index 4884c43614..237e8136df 100644 --- a/pkg/util/azure/classic_vpc.go +++ b/pkg/util/azure/classic_vpc.go @@ -5,6 +5,7 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/azure/classic_wire.go b/pkg/util/azure/classic_wire.go index b3c633d9df..81993dbe54 100644 --- a/pkg/util/azure/classic_wire.go +++ b/pkg/util/azure/classic_wire.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/azure/debug.go b/pkg/util/azure/debug.go index 57beb087d0..486ceefe38 100644 --- a/pkg/util/azure/debug.go +++ b/pkg/util/azure/debug.go @@ -4,9 +4,9 @@ import ( "net/http" "net/http/httputil" - "yunion.io/x/log" - "github.com/Azure/go-autorest/autorest" + + "yunion.io/x/log" ) const ( diff --git a/pkg/util/azure/disk.go b/pkg/util/azure/disk.go index 5a352360cd..886d476b98 100644 --- a/pkg/util/azure/disk.go +++ b/pkg/util/azure/disk.go @@ -1,13 +1,14 @@ package azure import ( + "context" "fmt" "strings" "time" - "context" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/eip.go b/pkg/util/azure/eip.go index 24d0a47f18..88fc697bc2 100644 --- a/pkg/util/azure/eip.go +++ b/pkg/util/azure/eip.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/host.go b/pkg/util/azure/host.go index 657f7ef10a..2ffd102ccc 100644 --- a/pkg/util/azure/host.go +++ b/pkg/util/azure/host.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/ansible" diff --git a/pkg/util/azure/image.go b/pkg/util/azure/image.go index 8c53edcefd..beb21825a1 100644 --- a/pkg/util/azure/image.go +++ b/pkg/util/azure/image.go @@ -1,11 +1,12 @@ package azure import ( + "context" "strings" - "context" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index ceade3ee7f..bbb935fef6 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -1,18 +1,18 @@ package azure import ( + "context" "fmt" "strings" "time" - "context" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" - "yunion.io/x/pkg/util/osprofile" ) const ( diff --git a/pkg/util/azure/instancenic.go b/pkg/util/azure/instancenic.go index fd1ac7787d..64f99daab3 100644 --- a/pkg/util/azure/instancenic.go +++ b/pkg/util/azure/instancenic.go @@ -5,8 +5,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type PublicIPAddress struct { diff --git a/pkg/util/azure/network.go b/pkg/util/azure/network.go index 692844d9b7..72eec45339 100644 --- a/pkg/util/azure/network.go +++ b/pkg/util/azure/network.go @@ -4,9 +4,10 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/util/netutils" ) type SNetwork struct { diff --git a/pkg/util/azure/region.go b/pkg/util/azure/region.go index 640d1eaac4..953ed21eae 100644 --- a/pkg/util/azure/region.go +++ b/pkg/util/azure/region.go @@ -5,10 +5,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/secrules" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/seclib2" - "yunion.io/x/pkg/util/secrules" ) type SVMSize struct { diff --git a/pkg/util/azure/snapshot.go b/pkg/util/azure/snapshot.go index ccc78658bd..72cc1a3cb4 100644 --- a/pkg/util/azure/snapshot.go +++ b/pkg/util/azure/snapshot.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/storage.go b/pkg/util/azure/storage.go index f77052649b..e1c2caa023 100644 --- a/pkg/util/azure/storage.go +++ b/pkg/util/azure/storage.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/azure/storageaccount.go b/pkg/util/azure/storageaccount.go index 622972b61c..8b146f0fcb 100644 --- a/pkg/util/azure/storageaccount.go +++ b/pkg/util/azure/storageaccount.go @@ -10,7 +10,9 @@ import ( "github.com/Azure/azure-sdk-for-go/storage" "github.com/Microsoft/azure-vhd-utils/vhdcore/common" "github.com/Microsoft/azure-vhd-utils/vhdcore/diskstream" + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/azure/storagecache.go b/pkg/util/azure/storagecache.go index eb6a7dcbc2..ecd854e364 100644 --- a/pkg/util/azure/storagecache.go +++ b/pkg/util/azure/storagecache.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" diff --git a/pkg/util/azure/upload.go b/pkg/util/azure/upload.go index 3910d7a4dd..0a51c84248 100644 --- a/pkg/util/azure/upload.go +++ b/pkg/util/azure/upload.go @@ -9,14 +9,14 @@ import ( "time" "github.com/Azure/azure-sdk-for-go/storage" - "yunion.io/x/onecloud/pkg/util/azure/concurrent" - "yunion.io/x/onecloud/pkg/util/azure/progress" - "github.com/Microsoft/azure-vhd-utils/vhdcore/block/bitmap" "github.com/Microsoft/azure-vhd-utils/vhdcore/common" "github.com/Microsoft/azure-vhd-utils/vhdcore/diskstream" "github.com/Microsoft/azure-vhd-utils/vhdcore/footer" "github.com/Microsoft/azure-vhd-utils/vhdcore/validator" + + "yunion.io/x/onecloud/pkg/util/azure/concurrent" + "yunion.io/x/onecloud/pkg/util/azure/progress" ) // DiskUploadContext type describes VHD upload context, this includes the disk stream to read from, the ranges of diff --git a/pkg/util/azure/vpc.go b/pkg/util/azure/vpc.go index ea02295e6f..2044db640c 100644 --- a/pkg/util/azure/vpc.go +++ b/pkg/util/azure/vpc.go @@ -4,6 +4,7 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/azure/wire.go b/pkg/util/azure/wire.go index 08830e8eeb..feb8901164 100644 --- a/pkg/util/azure/wire.go +++ b/pkg/util/azure/wire.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/azure/zone.go b/pkg/util/azure/zone.go index 17851ed4a5..981bf5e74c 100644 --- a/pkg/util/azure/zone.go +++ b/pkg/util/azure/zone.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/billing/billingcycle.go b/pkg/util/billing/billingcycle.go index 873bc513d1..d9b27dec6a 100644 --- a/pkg/util/billing/billingcycle.go +++ b/pkg/util/billing/billingcycle.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" "time" + "yunion.io/x/log" ) diff --git a/pkg/util/cloudinit/cloudconfig.go b/pkg/util/cloudinit/cloudconfig.go index b622846ca7..0c0711af5a 100644 --- a/pkg/util/cloudinit/cloudconfig.go +++ b/pkg/util/cloudinit/cloudconfig.go @@ -3,13 +3,14 @@ package cloudinit import ( "bytes" "encoding/base64" - "fmt" "strings" + "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/util/seclib2" "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/util/seclib2" ) /* diff --git a/pkg/util/conditionparser/parser.go b/pkg/util/conditionparser/parser.go index ef9cf967cb..7742a8d803 100644 --- a/pkg/util/conditionparser/parser.go +++ b/pkg/util/conditionparser/parser.go @@ -10,9 +10,8 @@ import ( "strings" "yunion.io/x/jsonutils" - "yunion.io/x/pkg/utils" - "yunion.io/x/log" + "yunion.io/x/pkg/utils" ) var ( diff --git a/pkg/util/esxi/datacenter.go b/pkg/util/esxi/datacenter.go index 1a6a5c0124..7d7c1359cd 100644 --- a/pkg/util/esxi/datacenter.go +++ b/pkg/util/esxi/datacenter.go @@ -4,7 +4,9 @@ import ( "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/esxi/device.go b/pkg/util/esxi/device.go index 4e2df97367..f80a764def 100644 --- a/pkg/util/esxi/device.go +++ b/pkg/util/esxi/device.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/vmware/govmomi/vim25/types" - // "yunion.io/x/log" ) type SVirtualDevice struct { diff --git a/pkg/util/esxi/image.go b/pkg/util/esxi/image.go index 378c2d2c5b..3788190d21 100644 --- a/pkg/util/esxi/image.go +++ b/pkg/util/esxi/image.go @@ -3,11 +3,12 @@ package esxi import ( "context" "path" + "strings" "github.com/vmware/govmomi/object" - "strings" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/esxi/storage.go b/pkg/util/esxi/storage.go index 34c786b937..e6e306bdc6 100644 --- a/pkg/util/esxi/storage.go +++ b/pkg/util/esxi/storage.go @@ -1,12 +1,8 @@ package esxi import ( - "github.com/vmware/govmomi/vim25/mo" - "context" "fmt" - "github.com/vmware/govmomi/object" - "github.com/vmware/govmomi/vim25/types" "io" "io/ioutil" "net/http" @@ -16,8 +12,14 @@ import ( "strconv" "strings" "time" + + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/vmdkutils" diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 998e22087a..49d1511eab 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "strings" "time" "github.com/vmware/govmomi/object" @@ -11,14 +12,12 @@ import ( "github.com/vmware/govmomi/vim25/types" "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" - "strings" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" ) diff --git a/pkg/util/esxi/vnic.go b/pkg/util/esxi/vnic.go index ae8196ea95..664124669c 100644 --- a/pkg/util/esxi/vnic.go +++ b/pkg/util/esxi/vnic.go @@ -4,8 +4,9 @@ import ( "github.com/vmware/govmomi/vim25/types" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SVirtualNIC struct { diff --git a/pkg/util/qcloud/disk.go b/pkg/util/qcloud/disk.go index 39ef816829..cf29ec44dd 100644 --- a/pkg/util/qcloud/disk.go +++ b/pkg/util/qcloud/disk.go @@ -1,17 +1,18 @@ package qcloud import ( + "context" "fmt" "sort" "strings" "time" - "context" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/utils" ) type Placement struct { diff --git a/pkg/util/qcloud/eip.go b/pkg/util/qcloud/eip.go index 10a2e00eaa..8334eaaaa2 100644 --- a/pkg/util/qcloud/eip.go +++ b/pkg/util/qcloud/eip.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/host.go b/pkg/util/qcloud/host.go index 8a9ce15ef9..167f27e7df 100644 --- a/pkg/util/qcloud/host.go +++ b/pkg/util/qcloud/host.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" diff --git a/pkg/util/qcloud/image.go b/pkg/util/qcloud/image.go index f99fbd651b..8ccc89dd8f 100644 --- a/pkg/util/qcloud/image.go +++ b/pkg/util/qcloud/image.go @@ -1,16 +1,17 @@ package qcloud import ( + "context" "fmt" "strconv" "strings" "time" - "context" "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type ImageStatusType string diff --git a/pkg/util/qcloud/instance.go b/pkg/util/qcloud/instance.go index 3df43b4773..e3b370ba64 100644 --- a/pkg/util/qcloud/instance.go +++ b/pkg/util/qcloud/instance.go @@ -8,10 +8,11 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/util/billing" - "yunion.io/x/pkg/utils" ) const ( diff --git a/pkg/util/qcloud/instancenic.go b/pkg/util/qcloud/instancenic.go index 82e36130dd..f9c71f4d27 100644 --- a/pkg/util/qcloud/instancenic.go +++ b/pkg/util/qcloud/instancenic.go @@ -1,8 +1,9 @@ package qcloud import ( - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SInstanceNic struct { diff --git a/pkg/util/qcloud/keypair.go b/pkg/util/qcloud/keypair.go index e6e31543f1..53f1df735e 100644 --- a/pkg/util/qcloud/keypair.go +++ b/pkg/util/qcloud/keypair.go @@ -7,7 +7,9 @@ import ( "github.com/aokoli/goutils" "golang.org/x/crypto/ssh" + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/qcloud/localdisk.go b/pkg/util/qcloud/localdisk.go index fcf7ddd06e..dd8d45b501 100644 --- a/pkg/util/qcloud/localdisk.go +++ b/pkg/util/qcloud/localdisk.go @@ -1,10 +1,11 @@ package qcloud import ( + "context" "time" - "context" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/localstorage.go b/pkg/util/qcloud/localstorage.go index 5d12e21564..9194805b87 100644 --- a/pkg/util/qcloud/localstorage.go +++ b/pkg/util/qcloud/localstorage.go @@ -5,6 +5,7 @@ import ( "strings" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/network.go b/pkg/util/qcloud/network.go index fa6ce8a6f2..fe7f7e1f54 100644 --- a/pkg/util/qcloud/network.go +++ b/pkg/util/qcloud/network.go @@ -5,9 +5,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/util/netutils" ) type SNetwork struct { diff --git a/pkg/util/qcloud/qcloud.go b/pkg/util/qcloud/qcloud.go index f35cfcd98f..7a1a5896a6 100644 --- a/pkg/util/qcloud/qcloud.go +++ b/pkg/util/qcloud/qcloud.go @@ -8,8 +8,10 @@ import ( "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/region.go b/pkg/util/qcloud/region.go index f9d57610df..eb8cd82330 100644 --- a/pkg/util/qcloud/region.go +++ b/pkg/util/qcloud/region.go @@ -2,11 +2,13 @@ package qcloud import ( "fmt" - "yunion.io/x/pkg/utils" "github.com/nelsonken/cos-go-sdk-v5/cos" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/securitygroup.go b/pkg/util/qcloud/securitygroup.go index 96b0f4ce62..2ec2733b26 100644 --- a/pkg/util/qcloud/securitygroup.go +++ b/pkg/util/qcloud/securitygroup.go @@ -10,9 +10,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/pkg/util/secrules" "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SecurityGroupPolicy struct { diff --git a/pkg/util/qcloud/snapshot.go b/pkg/util/qcloud/snapshot.go index 7f53083b0e..96fb56e61b 100644 --- a/pkg/util/qcloud/snapshot.go +++ b/pkg/util/qcloud/snapshot.go @@ -6,6 +6,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/storage.go b/pkg/util/qcloud/storage.go index 57acdc6709..e59680fedf 100644 --- a/pkg/util/qcloud/storage.go +++ b/pkg/util/qcloud/storage.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/storagecache.go b/pkg/util/qcloud/storagecache.go index 5ca144e310..da11f2d4f1 100644 --- a/pkg/util/qcloud/storagecache.go +++ b/pkg/util/qcloud/storagecache.go @@ -8,15 +8,16 @@ import ( "strings" "time" + coslib "github.com/nelsonken/cos-go-sdk-v5/cos" + "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" - - coslib "github.com/nelsonken/cos-go-sdk-v5/cos" ) type SStoragecache struct { diff --git a/pkg/util/qcloud/vpc.go b/pkg/util/qcloud/vpc.go index 654256320c..6598c41868 100644 --- a/pkg/util/qcloud/vpc.go +++ b/pkg/util/qcloud/vpc.go @@ -4,6 +4,7 @@ import ( "time" "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" ) diff --git a/pkg/util/qcloud/wire.go b/pkg/util/qcloud/wire.go index 41e331429d..c662f57e4e 100644 --- a/pkg/util/qcloud/wire.go +++ b/pkg/util/qcloud/wire.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudprovider" ) diff --git a/pkg/util/qcloud/zone.go b/pkg/util/qcloud/zone.go index 6f3f2f9f7d..76efc9b788 100644 --- a/pkg/util/qcloud/zone.go +++ b/pkg/util/qcloud/zone.go @@ -7,9 +7,10 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/pkg/utils" ) type InstanceChargeType string diff --git a/pkg/util/rbacutils/rabc.go b/pkg/util/rbacutils/rabc.go index 47c8ff6d69..2ad2461118 100644 --- a/pkg/util/rbacutils/rabc.go +++ b/pkg/util/rbacutils/rabc.go @@ -5,6 +5,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/util/conditionparser" ) diff --git a/pkg/util/vmdkutils/vmdkutils.go b/pkg/util/vmdkutils/vmdkutils.go index b08b45ecf9..b3c10940de 100644 --- a/pkg/util/vmdkutils/vmdkutils.go +++ b/pkg/util/vmdkutils/vmdkutils.go @@ -2,12 +2,12 @@ package vmdkutils import ( "bufio" + "fmt" "io" "regexp" + "strconv" "strings" - "fmt" - "strconv" "yunion.io/x/pkg/utils" ) From e7b978955c3a17712095964102afad558c68d2da Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 12 Dec 2018 23:34:52 +0800 Subject: [PATCH 34/34] set cmtbound for storage & cpu & memory --- pkg/compute/models/hosts.go | 2 ++ pkg/compute/models/storages.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 83edc18f70..03b657fcd7 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -1340,6 +1340,8 @@ func (manager *SHostManager) newFromCloudHost(extHost cloudprovider.ICloudHost, host.MemSize = extHost.GetMemSizeMB() host.StorageSize = extHost.GetStorageSizeMB() host.StorageType = extHost.GetStorageType() + host.CpuCmtbound = 8.0 + host.MemCmtbound = 1.0 host.ManagerId = extHost.GetManagerId() host.IsEmulated = extHost.IsEmulated() diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 709dcda1ce..20fe17a9b8 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -746,6 +746,7 @@ func (manager *SStorageManager) newFromCloudStorage(extStorage cloudprovider.ICl storage.MediumType = extStorage.GetMediumType() storage.StorageConf = extStorage.GetStorageConf() storage.Capacity = extStorage.GetCapacityMB() + storage.Cmtbound = 1.0 storage.Enabled = extStorage.GetEnabled()