diff --git a/Gopkg.lock b/Gopkg.lock index 35c2526f1c..4482016efa 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1124,7 +1124,7 @@ [[projects]] branch = "master" - digest = "1:253553e4aea4744856900c4015036177f56ddfa4400a4422a2aab8cb1737bd00" + digest = "1:e5dd1b8806ddb16ddd31b64c2208d0cc566cd184d12f63844c33581c28d7fa50" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1158,7 +1158,7 @@ "utils", ] pruneopts = "UT" - revision = "6a2c5061c44f5e4a813a87553014ef72d1b983ea" + revision = "98dfb17dd78a596fbe5542e8276f4c717dc752fe" [[projects]] branch = "master" diff --git a/cmd/climc/climc.go b/cmd/climc/climc.go index bc9618729f..f373f391cf 100644 --- a/cmd/climc/climc.go +++ b/cmd/climc/climc.go @@ -23,14 +23,14 @@ import ( ) type BaseOptions struct { - Help bool `help:"Show help" short-token:"h"` - Debug bool `help:"Show debug information"` - Version bool `help:"Show version"` - Timeout int `default:"600" help:"Number of seconds to wait for a response"` - Insecure bool `default:"false" help:"Allow skip server cert verification if URL is https" short-token:"k"` - NoCachedToken bool `default:"$NO_CACHED_TOKEN|false" help:"Force not use cached token"` - OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"` - OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"` + Help bool `help:"Show help" short-token:"h"` + Debug bool `help:"Show debug information"` + Version bool `help:"Show version"` + Timeout int `default:"600" help:"Number of seconds to wait for a response"` + Insecure bool `default:"false" help:"Allow skip server cert verification if URL is https" short-token:"k"` + UseCachedToken bool `default:"$YUNION_USE_CACHED_TOKEN|false" help:"Use cached token"` + OsUsername string `default:"$OS_USERNAME" help:"Username, defaults to env[OS_USERNAME]"` + OsPassword string `default:"$OS_PASSWORD" help:"Password, defaults to env[OS_PASSWORD]"` // OsProjectId string `default:"$OS_PROJECT_ID" help:"Proejct ID, defaults to env[OS_PROJECT_ID]"` OsProjectName string `default:"$OS_PROJECT_NAME" help:"Project name, defaults to env[OS_PROJECT_NAME]"` OsDomainName string `default:"$OS_DOMAIN_NAME" help:"Domain name, defaults to env[OS_DOMAIN_NAME]"` @@ -130,23 +130,24 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) { authUrlAlter := strings.Replace(options.OsAuthURL, "/", "", -1) authUrlAlter = strings.Replace(authUrlAlter, ":", "", -1) tokenCachePath := filepath.Join(os.TempDir(), fmt.Sprintf("OS_AUTH_CACHE_TOKEN-%s-%s-%s-%s", authUrlAlter, options.OsUsername, options.OsDomainName, options.OsProjectName)) - cacheFile, err := os.Open(tokenCachePath) - - if err == nil && cacheFile != nil && !options.NoCachedToken { - fileInfo, _ := cacheFile.Stat() - dur, err := time.ParseDuration("-24h") - if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) { - bytesToken, err := ioutil.ReadAll(cacheFile) - if err == nil { - token := client.NewAuthTokenCredential() - err := json.Unmarshal(bytesToken, token) - if err != nil { - fmt.Printf("Unmarshal token error:%s", err) - } else if token.IsValid() { - cacheToken = token + if options.UseCachedToken { + cacheFile, err := os.Open(tokenCachePath) + if err == nil && cacheFile != nil { + fileInfo, _ := cacheFile.Stat() + dur, err := time.ParseDuration("-24h") + if fileInfo != nil && err == nil && fileInfo.ModTime().After(time.Now().Add(dur)) { + bytesToken, err := ioutil.ReadAll(cacheFile) + if err == nil { + token := client.NewAuthTokenCredential() + err := json.Unmarshal(bytesToken, token) + if err != nil { + fmt.Printf("Unmarshal token error:%s", err) + } else if token.IsValid() { + cacheToken = token + } } + cacheFile.Close() } - cacheFile.Close() } } diff --git a/cmd/climc/shell/isolatedevices.go b/cmd/climc/shell/isolatedevices.go index 8b4c3e8da1..48dc75c5e9 100644 --- a/cmd/climc/shell/isolatedevices.go +++ b/cmd/climc/shell/isolatedevices.go @@ -13,6 +13,7 @@ func init() { Unused bool `help:"Only show unused devices"` Gpu bool `help:"Only show gpu devices"` Host string `help:"Host ID or Name"` + Zone string `help:"Zone ID or Name"` } R(&DeviceListOptions{}, "isolated-device-list", "List isolated devices like GPU", func(s *mcclient.ClientSession, args *DeviceListOptions) error { var params *jsonutils.JSONDict @@ -33,6 +34,9 @@ func init() { if args.Gpu { params.Add(jsonutils.JSONTrue, "gpu") } + if args.Zone != "" { + params.Add(jsonutils.NewString(args.Zone), "zone") + } result, err := modules.IsolatedDevices.List(s, params) if err != nil { return err diff --git a/cmd/climc/shell/res_results.go b/cmd/climc/shell/res_results.go new file mode 100644 index 0000000000..c64c29f218 --- /dev/null +++ b/cmd/climc/shell/res_results.go @@ -0,0 +1,90 @@ +package shell + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + + type ResResultsListOptions struct { + options.BaseListOptions + StatMonth string `help:"stat_month of the query"` + StartDate string `help:"start_date of the query"` + EndDate string `help:"end_date of the query"` + ProjectId string `help:"project_id of the query"` + } + R(&ResResultsListOptions{}, "resresult-list", "List all res results ", func(s *mcclient.ClientSession, args *ResResultsListOptions) error { + var params *jsonutils.JSONDict + { + var err error + params, err = args.BaseListOptions.Params() + if err != nil { + return err + + } + } + + if len(args.StatMonth) > 0 { + params.Add(jsonutils.NewString(args.StatMonth), "stat_month") + } + if len(args.StartDate) > 0 { + params.Add(jsonutils.NewString(args.StartDate), "start_date") + } + if len(args.EndDate) > 0 { + params.Add(jsonutils.NewString(args.EndDate), "end_date") + } + if len(args.ProjectId) > 0 { + params.Add(jsonutils.NewString(args.ProjectId), "project_id") + } + + result, err := modules.ResResults.List(s, params) + if err != nil { + return err + } + + printList(result, modules.ResResults.GetColumns(s)) + return nil + }) + + type ResResultUpdateOptions struct { + ID string `help:"ID of the query"` + StatMonth string `help:"stat_month of the query"` + StartDate string `help:"start_date of the query"` + EndDate string `help:"end_date of the query"` + ProjectId string `help:"project_id of the query"` + + ItemKey string `help:"item_key of the query"` + ItemText string `help:"item_text of the query"` + } + R(&ResResultUpdateOptions{}, "resresult-export", "Update a resresult export", func(s *mcclient.ClientSession, args *ResResultUpdateOptions) error { + params := jsonutils.NewDict() + if len(args.StatMonth) > 0 { + params.Add(jsonutils.NewString(args.StatMonth), "stat_month") + } + + if len(args.StartDate) > 0 { + params.Add(jsonutils.NewString(args.StartDate), "start_date") + } + if len(args.EndDate) > 0 { + params.Add(jsonutils.NewString(args.EndDate), "end_date") + } + if len(args.ProjectId) > 0 { + params.Add(jsonutils.NewString(args.ProjectId), "project_id") + } + if len(args.ItemKey) > 0 { + params.Add(jsonutils.NewString(args.ItemKey), "item_key") + } + if len(args.ItemText) > 0 { + params.Add(jsonutils.NewString(args.ItemText), "item_text") + } + resResult, err := modules.ResResults.Put(s, args.ID, params) + if err != nil { + return err + } + printObject(resResult) + return nil + }) +} diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index f3887dc575..6c956dc2b1 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -50,6 +50,11 @@ func init() { if err != nil { return err } + + if opts.NoAccountInit != nil && *opts.NoAccountInit { + params.Add(jsonutils.JSONFalse, "reset_password") + } + count := options.IntV(opts.Count) if options.BoolV(opts.DryRun) { results, err := modules.SchedManager.DoScheduleListResult(s, params, count) @@ -328,6 +333,11 @@ func init() { if err != nil { return err } + + if opts.NoAccountInit != nil && *opts.NoAccountInit { + params.Add(jsonutils.JSONFalse, "reset_password") + } + srv, err := modules.Servers.PerformAction(s, opts.ID, "rebuild-root", params) if err != nil { return err @@ -338,6 +348,13 @@ func init() { R(&options.ServerChangeConfigOptions{}, "server-change-config", "Change configuration of VM", func(s *mcclient.ClientSession, opts *options.ServerChangeConfigOptions) error { params, err := options.StructToParams(opts) + if len(opts.Disk) > 0 { + params.Remove("disk.0") + for i, d := range opts.Disk { + params.Set(fmt.Sprintf("disk.%d", i+1), jsonutils.NewString(d)) + } + } + if err != nil { return err } diff --git a/cmd/climc/shell/specs.go b/cmd/climc/shell/specs.go index e5f9094f7b..68f8f1e9aa 100644 --- a/cmd/climc/shell/specs.go +++ b/cmd/climc/shell/specs.go @@ -15,6 +15,7 @@ func init() { Model string `help:"Specified model specs" choices:"hosts|isolated_devices|guests"` HostType string `help:"Host type filter" choices:"baremetal|hypervisor|esxi|kubelet|hyperv"` Gpu bool `help:"Only show gpu devices"` + Zone string `help:"Filter by zone id or name"` } R(&ListOptions{}, "spec", "List all kinds of model specs", func(s *mcclient.ClientSession, args *ListOptions) error { var params *jsonutils.JSONDict @@ -36,6 +37,9 @@ func init() { if args.Gpu { params.Add(jsonutils.JSONTrue, "gpu") } + if args.Zone != "" { + params.Add(jsonutils.NewString(args.Zone), "zone") + } result, err := modules.Specs.GetModelSpecs(s, model, params) if err != nil { return err diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index b7435030dd..49620e8fc8 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -254,6 +254,11 @@ func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery return q, nil } +func ListItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + return listItemQueryFilters(manager, ctx, q, userCred, query) +} + func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { @@ -954,10 +959,6 @@ func objectPerformAction(dispatcher *DBModelDispatcher, modelValue reflect.Value } func updateItem(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - if !item.AllowUpdateItem(ctx, userCred) { - return nil, httperrors.NewForbiddenError(fmt.Sprintf("Not allow to update item")) - } - var err error err = item.ValidateUpdateCondition(ctx) @@ -1029,6 +1030,10 @@ func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, q return nil, httperrors.NewGeneralError(err) } + if !model.AllowUpdateItem(ctx, userCred) { + return nil, httperrors.NewForbiddenError(fmt.Sprintf("Not allow to update item")) + } + lockman.LockObject(ctx, model) defer lockman.ReleaseObject(ctx, model) diff --git a/pkg/cloudcommon/db/db_joint_dispatcher.go b/pkg/cloudcommon/db/db_joint_dispatcher.go index 8b287f9544..236d6104f6 100644 --- a/pkg/cloudcommon/db/db_joint_dispatcher.go +++ b/pkg/cloudcommon/db/db_joint_dispatcher.go @@ -136,7 +136,7 @@ func (dispatcher *DBJointModelDispatcher) Get(ctx context.Context, id1 string, i } else if err != nil { return nil, httperrors.NewGeneralError(err) } - if !item.AllowGetDetails(ctx, userCred, query) { + if !item.AllowGetJointDetails(ctx, userCred, query, item) { return nil, httperrors.NewForbiddenError("Not allow to get details") } return getItemDetails(dispatcher.JointModelManager(), item, ctx, userCred, query) @@ -200,6 +200,11 @@ func (dispatcher *DBJointModelDispatcher) Update(ctx context.Context, id1 string } else if err != nil { return nil, httperrors.NewGeneralError(err) } + + if !item.AllowUpdateJointItem(ctx, userCred, item) { + return nil, httperrors.NewForbiddenError(fmt.Sprintf("Not allow to update item")) + } + lockman.LockJointObject(ctx, master, slave) defer lockman.ReleaseJointObject(ctx, master, slave) return updateItem(dispatcher.JointModelManager(), item, ctx, userCred, query, data) diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index bcc4c00779..4b0a7a3ac5 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -131,6 +131,8 @@ type IJointModel interface { Slave() IStandaloneModel Detach(ctx context.Context, userCred mcclient.TokenCredential) error + AllowGetJointDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, item IJointModel) bool + AllowUpdateJointItem(ctx context.Context, userCred mcclient.TokenCredential, item IJointModel) bool } type IStandaloneModelManager interface { diff --git a/pkg/cloudcommon/db/jointbase.go b/pkg/cloudcommon/db/jointbase.go index b8c3bf4ef0..3679eea408 100644 --- a/pkg/cloudcommon/db/jointbase.go +++ b/pkg/cloudcommon/db/jointbase.go @@ -179,6 +179,16 @@ func (joint *SJointResourceBase) Slave() IStandaloneModel { return nil } +func (self *SJointResourceBase) AllowGetJointDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, item IJointModel) bool { + masterVirtual := item.Master().(IVirtualModel) + return masterVirtual.IsOwner(userCred) +} + +func (self *SJointResourceBase) AllowUpdateJointItem(ctx context.Context, userCred mcclient.TokenCredential, item IJointModel) bool { + masterVirtual := item.Master().(IVirtualModel) + return masterVirtual.IsOwner(userCred) +} + /* func (joint *SJointResourceBase) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { extra := joint.SResourceBase.GetCustomizeColumns(ctx, userCred, query) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 1c59635c80..51e99193cd 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -21,6 +21,11 @@ type ICloudResource interface { GetMetadata() *jsonutils.JSONDict } +type IBillingResource interface { + GetBillingType() string + GetExpiredAt() time.Time +} + type ICloudRegion interface { ICloudResource @@ -133,6 +138,7 @@ type ICloudHost interface { type ICloudVM interface { ICloudResource + IBillingResource GetCreateTime() time.Time GetIHost() ICloudHost @@ -213,6 +219,7 @@ type ICloudSecurityGroup interface { type ICloudDisk interface { ICloudResource + IBillingResource GetIStorge() ICloudStorage diff --git a/pkg/scheduler/util/baremetal/diskconfig.go b/pkg/compute/baremetal/diskconfig.go similarity index 79% rename from pkg/scheduler/util/baremetal/diskconfig.go rename to pkg/compute/baremetal/diskconfig.go index 0914e4ec81..a7a28e5ce0 100644 --- a/pkg/scheduler/util/baremetal/diskconfig.go +++ b/pkg/compute/baremetal/diskconfig.go @@ -8,68 +8,10 @@ import ( "strings" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" ) -const ( - DISK_CONF_RAID0 = api.DISK_CONF_RAID0 - DISK_CONF_RAID1 = api.DISK_CONF_RAID1 - DISK_CONF_RAID5 = api.DISK_CONF_RAID5 - DISK_CONF_RAID10 = api.DISK_CONF_RAID10 - DISK_CONF_NONE = api.DISK_CONF_NONE - - DEFAULT_DISK_CONF = DISK_CONF_NONE - - DISK_TYPE_ROTATE = api.DISK_TYPE_ROTATE - DISK_TYPE_SSD = api.DISK_TYPE_SSD - DISK_TYPE_HYBRID = api.DISK_TYPE_HYBRID - - DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE - - DISK_DRIVER_MEGARAID = api.DISK_DRIVER_MEGARAID - DISK_DRIVER_LINUX = api.DISK_DRIVER_LINUX - DISK_DRIVER_HPSARAID = api.DISK_DRIVER_HPSARAID - DISK_DRIVER_MPT2SAS = api.DISK_DRIVER_MPT2SAS - DISK_DRIVER_MARVELRAID = api.DISK_DRIVER_MARVELRAID - DISK_DRIVER_PCIE = api.DISK_DRIVER_PCIE - - HDD_DISK_SPEC_TYPE = api.HDD_DISK_SPEC_TYPE - SSD_DISK_SPEC_TYPE = api.SSD_DISK_SPEC_TYPE - - ErrMoreThanOneSizeUnspecificSplit = `more than 1 size unspecific split` - ErrNoMoreSpaceForUnspecificSplit = `no more space for an unspecific split` - ErrSubtotalOfSplitExceedsDiskSize = `subtotal of split exceeds disk size` -) - -var ( - DISK_CONFS = sets.NewString( - DISK_CONF_RAID0, - DISK_CONF_RAID1, - DISK_CONF_RAID5, - DISK_CONF_RAID10, - DISK_CONF_NONE, - ) - - DISK_TYPES = sets.NewString( - DISK_TYPE_ROTATE, - DISK_TYPE_SSD, - DISK_TYPE_HYBRID, - ) - - DISK_DRIVERS_RAID = sets.NewString( - DISK_DRIVER_MEGARAID, - DISK_DRIVER_HPSARAID, - DISK_DRIVER_MPT2SAS, - DISK_DRIVER_MARVELRAID, - ) - - DISK_DRIVERS = sets.NewString( - DISK_DRIVER_LINUX, - DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID) -) - // return bytes func parseStrip(stripStr string, defaultSize string) int64 { size, _ := utils.GetSize(stripStr, defaultSize, 1024) @@ -149,7 +91,7 @@ func ParseRange(rangeStr string) (ret []int64, err error) { return } -func ParseDiskConfig(desc string) (bdc api.BaremetalDiskConfig, err error) { +func ParseDiskConfig(desc string) (bdc BaremetalDiskConfig, err error) { bdc.Type = DISK_TYPE_HYBRID bdc.Conf = DISK_CONF_NONE bdc.Count = 0 @@ -216,11 +158,11 @@ func ParseDiskConfig(desc string) (bdc api.BaremetalDiskConfig, err error) { } func isDiskConfigStorageMatch( - config *api.BaremetalDiskConfig, + config *BaremetalDiskConfig, confDriver *string, confAdapter *int, - storage *api.BaremetalStorage, - selected []*api.BaremetalStorage, + storage *BaremetalStorage, + selected []*BaremetalStorage, ) bool { isRotate := storage.Rotate adapter := storage.Adapter @@ -252,7 +194,7 @@ func isDiskConfigStorageMatch( return false } -func RetrieveStorages(diskConfig *api.BaremetalDiskConfig, storages []*api.BaremetalStorage) (selected, rest []*api.BaremetalStorage) { +func RetrieveStorages(diskConfig *BaremetalDiskConfig, storages []*BaremetalStorage) (selected, rest []*BaremetalStorage) { var confDriver *string = nil var confAdapter *int = nil @@ -263,8 +205,8 @@ func RetrieveStorages(diskConfig *api.BaremetalDiskConfig, storages []*api.Barem confDriver = &diskConfig.Driver } - selected = make([]*api.BaremetalStorage, 0) - rest = make([]*api.BaremetalStorage, 0) + selected = make([]*BaremetalStorage, 0) + rest = make([]*BaremetalStorage, 0) idx := 0 for _, storage := range storages { @@ -307,9 +249,9 @@ func RequireEvenDisks(diskConfig string) bool { } type Layout struct { - Disks []*api.BaremetalStorage `json:"disks"` - Conf *api.BaremetalDiskConfig `json:"conf"` - Size int64 `json:"size"` + Disks []*BaremetalStorage `json:"disks"` + Conf *BaremetalDiskConfig `json:"conf"` + Size int64 `json:"size"` } func (l Layout) String() string { @@ -317,7 +259,7 @@ func (l Layout) String() string { return string(bytes) } -func RetrieveStorageDrivers(storages []*api.BaremetalStorage) sets.String { +func RetrieveStorageDrivers(storages []*BaremetalStorage) sets.String { ret := sets.NewString() for _, s := range storages { ret = ret.Union(sets.NewString(s.Driver)) @@ -326,8 +268,8 @@ func RetrieveStorageDrivers(storages []*api.BaremetalStorage) sets.String { } func MeetConfig( - conf *api.BaremetalDiskConfig, - storages []*api.BaremetalStorage, + conf *BaremetalDiskConfig, + storages []*BaremetalStorage, ) error { storageDrvs := RetrieveStorageDrivers(storages) if len(storageDrvs.List()) > 1 { @@ -380,7 +322,7 @@ func MeetConfig( return nil } -func GetStoragesMinSize(ss []*api.BaremetalStorage) int64 { +func GetStoragesMinSize(ss []*BaremetalStorage) int64 { minSize := int64(-1) for _, s := range ss { if minSize < 0 || minSize > s.Size { @@ -390,7 +332,7 @@ func GetStoragesMinSize(ss []*api.BaremetalStorage) int64 { return minSize } -func CalculateSize(conf string, storages []*api.BaremetalStorage) int64 { +func CalculateSize(conf string, storages []*BaremetalStorage) int64 { if conf == "" { conf = DEFAULT_DISK_CONF } @@ -427,7 +369,7 @@ func GetSplitSizes(size int64, splitConf string) []int64 { subtotal += isizes[index] } else { if leftoverIdx >= 0 { - log.Errorln(ErrMoreThanOneSizeUnspecificSplit) + log.Errorf("%v", ErrMoreThanOneSizeUnspecificSplit) return []int64{} } leftoverIdx = index @@ -436,12 +378,12 @@ func GetSplitSizes(size int64, splitConf string) []int64 { if leftoverIdx >= 0 { isizes[leftoverIdx] = size - subtotal if isizes[leftoverIdx] <= 0 { - log.Errorln(ErrNoMoreSpaceForUnspecificSplit) + log.Errorf("%v", ErrNoMoreSpaceForUnspecificSplit) return []int64{} } } else { if subtotal > size { - log.Errorln(ErrSubtotalOfSplitExceedsDiskSize) + log.Errorf("%v", ErrSubtotalOfSplitExceedsDiskSize) return []int64{} } } @@ -454,7 +396,7 @@ func ExpandNoneConf(layouts []Layout) (ret []Layout) { conf := layout.Conf conf.Count = 1 for _, disk := range layout.Disks { - ret = append(ret, Layout{Disks: []*api.BaremetalStorage{disk}, Conf: conf, Size: disk.Size}) + ret = append(ret, Layout{Disks: []*BaremetalStorage{disk}, Conf: conf, Size: disk.Size}) } } else { ret = append(ret, layout) @@ -464,20 +406,20 @@ func ExpandNoneConf(layouts []Layout) (ret []Layout) { } func CalculateLayout( - confs []*api.BaremetalDiskConfig, - storages []*api.BaremetalStorage, + confs []*BaremetalDiskConfig, + storages []*BaremetalStorage, ) (layouts []Layout, err error) { if len(confs) == 0 { - err = fmt.Errorf("[]*api.BaremetalDiskConfig must be provided.") + err = fmt.Errorf("[]*BaremetalDiskConfig must be provided.") return } layouts = make([]Layout, 0) var ( - plainDisks = make([]*api.BaremetalStorage, 0) - pcieDisks = make([]*api.BaremetalStorage, 0) - raidDisks = make([]*api.BaremetalStorage, 0) + plainDisks = make([]*BaremetalStorage, 0) + pcieDisks = make([]*BaremetalStorage, 0) + raidDisks = make([]*BaremetalStorage, 0) ) for _, storage := range storages { @@ -493,7 +435,7 @@ func CalculateLayout( if len(plainDisks) > 0 { layouts = append(layouts, Layout{ Disks: plainDisks, - Conf: &api.BaremetalDiskConfig{Conf: DISK_CONF_NONE}, + Conf: &BaremetalDiskConfig{Conf: DISK_CONF_NONE}, Size: 0, }) } @@ -557,7 +499,7 @@ func CalculateLayout( if len(pcieDisks) > 0 { layouts = append(layouts, Layout{ Disks: pcieDisks, - Conf: &api.BaremetalDiskConfig{Conf: DISK_CONF_NONE}, + Conf: &BaremetalDiskConfig{Conf: DISK_CONF_NONE}, Size: 0, }) } @@ -582,7 +524,7 @@ func expandLayoutSplits(layouts []Layout) []Layout { return ret } -func CheckDisksAllocable(layouts []Layout, disks []*api.Disk) bool { +func CheckDisksAllocable(layouts []Layout, disks []*Disk) bool { layouts = expandLayoutSplits(layouts) storeIndex := 0 storeFreeSize := int64(-1) @@ -619,8 +561,8 @@ func CheckDisksAllocable(layouts []Layout, disks []*api.Disk) bool { return true } -func NewBaremetalDiskConfigs(dss ...string) ([]*api.BaremetalDiskConfig, error) { - ret := make([]*api.BaremetalDiskConfig, 0) +func NewBaremetalDiskConfigs(dss ...string) ([]*BaremetalDiskConfig, error) { + ret := make([]*BaremetalDiskConfig, 0) for _, ds := range dss { r, err := ParseDiskConfig(ds) if err != nil { @@ -634,7 +576,7 @@ func NewBaremetalDiskConfigs(dss ...string) ([]*api.BaremetalDiskConfig, error) type SpecSizeCount map[string]int type DiskSpec map[string]SpecSizeCount -func GetDiskSpec(storages []*api.BaremetalStorage) DiskSpec { +func GetDiskSpec(storages []*BaremetalStorage) DiskSpec { diskSpec := make(map[string]SpecSizeCount) for _, s := range storages { @@ -658,3 +600,56 @@ func GetDiskSpec(storages []*api.BaremetalStorage) DiskSpec { } return diskSpec } + +func getStoragesByDriver(driver string, storages []*BaremetalStorage) []*BaremetalStorage { + ret := make([]*BaremetalStorage, 0) + for _, s := range storages { + if s.Driver == driver { + ret = append(ret, s) + } + } + return ret +} + +func groupByAdapter(storages []*BaremetalStorage) map[string][]*BaremetalStorage { + ret := make(map[string][]*BaremetalStorage) + for _, storage := range storages { + adapter := storage.Adapter + adapterKey := fmt.Sprintf("adapter%d", adapter) + oldStorages := ret[adapterKey] + if len(oldStorages) == 0 { + ret[adapterKey] = []*BaremetalStorage{storage} + } else { + oldStorages = append(oldStorages, storage) + ret[adapterKey] = oldStorages + } + } + return ret +} + +type DiskAdapterSpecs map[string]DiskSpec + +func getSpec(storages []*BaremetalStorage) DiskAdapterSpecs { + ret := make(map[string]DiskSpec) + for adapterKey, newStorages := range groupByAdapter(storages) { + if len(newStorages) == 0 { + continue + } + ret[adapterKey] = GetDiskSpec(newStorages) + } + return ret +} + +type DiskDriverSpecs map[string]DiskAdapterSpecs + +func GetDiskSpecV2(storages []*BaremetalStorage) DiskDriverSpecs { + spec := make(map[string]DiskAdapterSpecs) + for _, driver := range DISK_DRIVERS.List() { + driverStorages := getStoragesByDriver(driver, storages) + if len(driverStorages) == 0 { + continue + } + spec[driver] = getSpec(storages) + } + return spec +} diff --git a/pkg/scheduler/util/baremetal/diskconfig_test.go b/pkg/compute/baremetal/diskconfig_test.go similarity index 96% rename from pkg/scheduler/util/baremetal/diskconfig_test.go rename to pkg/compute/baremetal/diskconfig_test.go index 2bda6f4ca3..b5bd96d0ea 100644 --- a/pkg/scheduler/util/baremetal/diskconfig_test.go +++ b/pkg/compute/baremetal/diskconfig_test.go @@ -6,7 +6,6 @@ import ( "testing" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/scheduler/api" ) func TestParseDiskConfig(t *testing.T) { @@ -23,13 +22,13 @@ func TestParseDiskConfig(t *testing.T) { tests := []struct { name string args args - wantBdc api.BaremetalDiskConfig + wantBdc BaremetalDiskConfig wantErr bool }{ { name: "rotate:[1-2,4-5]:MegaRaid", args: args{"rotate:[1-2,4-5]:MegaRaid"}, - wantBdc: api.BaremetalDiskConfig{ + wantBdc: BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_NONE, Driver: DISK_DRIVER_MEGARAID, @@ -41,7 +40,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[1-2,4,6]:raid10:marvelraid", args: args{"rotate:[1-2,4,6]:raid10:marvelraid"}, - wantBdc: api.BaremetalDiskConfig{ + wantBdc: BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Driver: DISK_DRIVER_MARVELRAID, @@ -53,7 +52,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[4,6]:raid10", args: args{"rotate:[4,6]:raid10"}, - wantBdc: api.BaremetalDiskConfig{ + wantBdc: BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Count: 0, @@ -64,7 +63,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "rotate:[4]:raid10:(40%, )", args: args{"rotate:[4]:raid10:(40%, )"}, - wantBdc: api.BaremetalDiskConfig{ + wantBdc: BaremetalDiskConfig{ Type: DISK_TYPE_ROTATE, Conf: DISK_CONF_RAID10, Count: 0, @@ -76,7 +75,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "[12-13]:raid1:(100g,32g,):adapter0:strip64k", args: args{"[12-13]:raid1:(100g,32g,):adapter0:strip64k"}, - wantBdc: api.BaremetalDiskConfig{ + wantBdc: BaremetalDiskConfig{ Type: DISK_TYPE_HYBRID, Conf: DISK_CONF_RAID1, Count: 0, @@ -90,7 +89,7 @@ func TestParseDiskConfig(t *testing.T) { { name: "6:raid5:adapter2", args: args{"6:raid5:adapter2"}, - wantBdc: api.BaremetalDiskConfig{ + wantBdc: BaremetalDiskConfig{ Type: DISK_TYPE_HYBRID, Conf: DISK_CONF_RAID5, Count: 6, @@ -163,7 +162,7 @@ func TestParseRange(t *testing.T) { } var ( - testStorages = []*api.BaremetalStorage{ + testStorages = []*BaremetalStorage{ // 0-11 disk on adapter2 { Driver: DISK_DRIVER_MEGARAID, @@ -252,7 +251,7 @@ var ( Adapter: 0, }, } - bitmainStorage = []*api.BaremetalStorage{ + bitmainStorage = []*BaremetalStorage{ { Driver: DISK_DRIVER_MARVELRAID, Rotate: true, @@ -580,7 +579,7 @@ func TestCheckDisksAllocable(t *testing.T) { } layout, err := CalculateLayout(confs, testStorages) - defaultLayout, err := CalculateLayout([]*api.BaremetalDiskConfig{&api.BaremetalDefaultDiskConfig}, testStorages[12:]) + defaultLayout, err := CalculateLayout([]*BaremetalDiskConfig{&BaremetalDefaultDiskConfig}, testStorages[12:]) if err != nil { t.Fatalf("CalculateLayout err: %v", err) } @@ -593,48 +592,48 @@ func TestCheckDisksAllocable(t *testing.T) { log.Debugf("layout: %s", layout) log.Debugf("Bitmain layout: %s", bitmainLayout) - tdiskDefault := []*api.Disk{ + tdiskDefault := []*Disk{ {Size: -1}, {Size: -1}, } - tdisk1 := []*api.Disk{ + tdisk1 := []*Disk{ {Size: 960000}, {Size: -1}, {Size: -1}, {Size: -1}, } - tdisk2 := []*api.Disk{ + tdisk2 := []*Disk{ {Size: -1}, {Size: -1}, {Size: -1}, {Size: -1}, } - tdisk3 := []*api.Disk{ + tdisk3 := []*Disk{ {Size: 102398}, {Size: -1}, {Size: -1}, {Size: -1}, } - btdisk1 := []*api.Disk{ + btdisk1 := []*Disk{ {Size: 61438}, {Size: -1}, } - btdisk2 := []*api.Disk{ + btdisk2 := []*Disk{ {Size: 61440}, {Size: -1}, } - btdisk3 := []*api.Disk{ + btdisk3 := []*Disk{ {Size: -1}, {Size: -1}, } - btPcieDisk := []*api.Disk{ + btPcieDisk := []*Disk{ {Size: 44440}, {Size: -1}, {Size: -1}, @@ -644,7 +643,7 @@ func TestCheckDisksAllocable(t *testing.T) { type args struct { layouts []Layout - disks []*api.Disk + disks []*Disk } tests := []struct { name string @@ -727,7 +726,7 @@ func TestCheckDisksAllocable(t *testing.T) { func TestGetDiskSpec(t *testing.T) { type args struct { - storages []*api.BaremetalStorage + storages []*BaremetalStorage } tests := []struct { name string @@ -1068,7 +1067,7 @@ var testStorages2 string = ` ` func TestStorageLoad(t *testing.T) { - ss := make([]*api.BaremetalStorage, 0) + ss := make([]*BaremetalStorage, 0) confs, err := NewBaremetalDiskConfigs( "2:raid1:MarvelRaid", "4:raid10:MegaRaid", diff --git a/pkg/compute/baremetal/doc.go b/pkg/compute/baremetal/doc.go new file mode 100644 index 0000000000..c9baa72a40 --- /dev/null +++ b/pkg/compute/baremetal/doc.go @@ -0,0 +1 @@ +package baremetal // import "yunion.io/x/onecloud/pkg/compute/baremetal" diff --git a/pkg/compute/baremetal/types.go b/pkg/compute/baremetal/types.go new file mode 100644 index 0000000000..dcf6bf5b4f --- /dev/null +++ b/pkg/compute/baremetal/types.go @@ -0,0 +1,119 @@ +package baremetal + +import ( + "errors" + + "yunion.io/x/pkg/util/sets" +) + +const ( + DISK_CONF_RAID0 = "raid0" + DISK_CONF_RAID1 = "raid1" + DISK_CONF_RAID5 = "raid5" + DISK_CONF_RAID10 = "raid10" + DISK_CONF_NONE = "none" + + DEFAULT_DISK_CONF = DISK_CONF_NONE + + DISK_TYPE_ROTATE = "rotate" + DISK_TYPE_SSD = "ssd" + DISK_TYPE_HYBRID = "hybrid" + + DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE + + DISK_DRIVER_MEGARAID = "MegaRaid" + DISK_DRIVER_LINUX = "Linux" + DISK_DRIVER_HPSARAID = "HPSARaid" + DISK_DRIVER_MPT2SAS = "Mpt2SAS" + DISK_DRIVER_MARVELRAID = "MarvelRaid" + DISK_DRIVER_PCIE = "PCIE" + + HDD_DISK_SPEC_TYPE = "HDD" + SSD_DISK_SPEC_TYPE = "SSD" +) + +var ( + BaremetalDefaultDiskConfig = BaremetalDiskConfig{ + Type: DISK_TYPE_HYBRID, + Conf: DISK_CONF_NONE, + Count: 0, + } + + DISK_CONFS = sets.NewString( + DISK_CONF_RAID0, + DISK_CONF_RAID1, + DISK_CONF_RAID5, + DISK_CONF_RAID10, + DISK_CONF_NONE, + ) + + DISK_TYPES = sets.NewString( + DISK_TYPE_ROTATE, + DISK_TYPE_SSD, + DISK_TYPE_HYBRID, + ) + + DISK_DRIVERS_RAID = sets.NewString( + DISK_DRIVER_MEGARAID, + DISK_DRIVER_HPSARAID, + DISK_DRIVER_MPT2SAS, + DISK_DRIVER_MARVELRAID, + ) + + DISK_DRIVERS = sets.NewString( + DISK_DRIVER_LINUX, + DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID) +) + +var ( + ErrMoreThanOneSizeUnspecificSplit = errors.New(`more than 1 size unspecific split`) + ErrNoMoreSpaceForUnspecificSplit = errors.New(`no more space for an unspecific split`) + ErrSubtotalOfSplitExceedsDiskSize = errors.New(`subtotal of split exceeds disk size`) +) + +type BaremetalStorage struct { + Slot int `json:"slot"` + Status string `json:"status"` + Rotate bool `json:"rotate"` + Adapter int `json:"adapter"` + Driver string `json:"driver"` + Model string `json:"model"` + Enclosure int `json:"enclousure"` + Size int64 `json:"size"` + MinStripSize int64 `json:"min_strip_size,omitempty"` + MaxStripSize int64 `json:"max_strip_size,omitempty"` + Index int64 `json:"index"` +} + +type BaremetalDiskConfig struct { + // disk type + Type string `json:"type"` + // raid config + Conf string `json:"conf"` + Count int64 `json:"count"` + Range []int64 `json:"range"` + Splits string `json:"splits"` + Adapter *int `json:"adapter"` + Driver string `json:"driver"` + Cachedbadbbu bool `json:"cachedbadbbu"` + Strip int64 `json:"strip"` + RA bool `json:"ra"` + WT bool `json:"wt"` + Direct bool `json:"direct"` +} + +type Disk struct { + Backend string `json:"backend"` + ImageID string `json:"image_id"` + Fs *string `json:"fs"` + Os string `json:"os"` + OSDistribution string `json:"os_distribution"` + OsVersion string `json:"os_version"` + Format string `json:"format"` + MountPoint *string `json:"mountpoint"` + Driver *string `json:"driver"` + Cache *string `json:"cache"` + ImageDiskFormat string `json:"image_disk_format"` + Size int64 `json:"size"` + Storage *string `json:"storage"` +} diff --git a/pkg/compute/capabilities/handler.go b/pkg/compute/capabilities/handler.go index 0a72df205f..a6f0aa9378 100644 --- a/pkg/compute/capabilities/handler.go +++ b/pkg/compute/capabilities/handler.go @@ -9,6 +9,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient/auth" ) @@ -16,7 +17,17 @@ func AddCapabilityHandler(prefix string, app *appsrv.Application) { app.AddHandler2("GET", fmt.Sprintf("%s/capabilities", prefix), auth.Authenticate(capaHandler), nil, "get_capabilities", nil) } -func capaHandler(context context.Context, w http.ResponseWriter, r *http.Request) { - capa := models.GetCapabilities(nil) +func capaHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + userCred := auth.FetchUserCredential(ctx) + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } + capa, err := models.GetCapabilities(ctx, userCred, query, nil) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } appsrv.SendJSON(w, jsonutils.Marshal(capa)) } diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 19368731b6..7aa3d7fa98 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -266,21 +266,18 @@ func (self *SKVMGuestDriver) RequestDeleteDetachedDisk(ctx context.Context, disk } func (self *SKVMGuestDriver) RequestSyncConfigOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, task taskman.ITask) error { - if guest.Status == models.VM_RUNNING { - desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host) - body := jsonutils.NewDict() - body.Add(desc, "desc") - if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { - body.Add(jsonutils.JSONTrue, "fw_only") - } - url := fmt.Sprintf("/servers/%s/sync", guest.Id) - header := http.Header{} - header.Add("X-Task-Id", task.GetTaskId()) - header.Add("X-Region-Version", "v2") - _, err := host.Request(task.GetUserCred(), "POST", url, header, body) - return err + desc := guest.GetDriver().GetJsonDescAtHost(ctx, guest, host) + body := jsonutils.NewDict() + body.Add(desc, "desc") + if fw_only, _ := task.GetParams().Bool("fw_only"); fw_only { + body.Add(jsonutils.JSONTrue, "fw_only") } - return nil + url := fmt.Sprintf("/servers/%s/sync", guest.Id) + header := http.Header{} + header.Add("X-Task-Id", task.GetTaskId()) + header.Add("X-Region-Version", "v2") + _, err := host.Request(task.GetUserCred(), "POST", url, header, body) + return err } func (self *SKVMGuestDriver) RqeuestSuspendOnHost(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { diff --git a/pkg/compute/handlers.go b/pkg/compute/handlers.go index 8e2864bc6f..311109892f 100644 --- a/pkg/compute/handlers.go +++ b/pkg/compute/handlers.go @@ -10,6 +10,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/capabilities" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/compute/specs" "yunion.io/x/onecloud/pkg/compute/usages" ) @@ -19,6 +20,7 @@ func InitHandlers(app *appsrv.Application) { quotas.AddQuotaHandler(models.QuotaManager, "", app) usages.AddUsageHandler("", app) capabilities.AddCapabilityHandler("", app) + specs.AddSpecHandler("", app) taskman.AddTaskHandler("", app) diff --git a/pkg/compute/models/billingresource.go b/pkg/compute/models/billingresource.go new file mode 100644 index 0000000000..612919afdc --- /dev/null +++ b/pkg/compute/models/billingresource.go @@ -0,0 +1,21 @@ +package models + +import "time" + +const ( + BILLING_TYPE_POSTPAID = "postpaid" + BILLING_TYPE_PREPAID = "prepaid" +) + +type SBillingResourceBase struct { + BillingType string `width:"36" charset:"ascii" nullable:"true" default:"postpaid" list:"user" create:"optional"` + ExpiredAt time.Time `nullable:"true" list:"user" create:"optional"` +} + +func (self *SBillingResourceBase) GetChargeType() string { + if len(self.BillingType) > 0 { + return self.BillingType + } else { + return BILLING_TYPE_POSTPAID + } +} diff --git a/pkg/compute/models/capabilities.go b/pkg/compute/models/capabilities.go index d6a3423234..c85fe238ba 100644 --- a/pkg/compute/models/capabilities.go +++ b/pkg/compute/models/capabilities.go @@ -1,9 +1,13 @@ package models import ( + "context" "fmt" + "yunion.io/x/jsonutils" "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/mcclient" ) type SCapabilities struct { @@ -16,9 +20,10 @@ type SCapabilities struct { MaxDataDiskCount int SchedPolicySupport bool Usable bool + Specs jsonutils.JSONObject } -func GetCapabilities(zone *SZone) SCapabilities { +func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, zone *SZone) (SCapabilities, error) { capa := SCapabilities{} capa.Hypervisors = getHypervisors(zone) capa.StorageTypes = getStorageTypes(zone) @@ -29,7 +34,16 @@ func GetCapabilities(zone *SZone) SCapabilities { capa.MinDataDiskCount = getMinDataDiskCount(zone) capa.MaxDataDiskCount = getMaxDataDiskCount(zone) capa.Usable = isUsable(zone) - return capa + if query == nil { + query = jsonutils.NewDict() + } + var err error + if zone != nil { + query.(*jsonutils.JSONDict).Add(jsonutils.NewString(zone.GetId()), "zone") + } + mans := []ISpecModelManager{HostManager, IsolatedDeviceManager} + capa.Specs, err = GetModelsSpecs(ctx, userCred, query.(*jsonutils.JSONDict), mans...) + return capa, err } func getHypervisors(zone *SZone) []string { @@ -37,6 +51,7 @@ func getHypervisors(zone *SZone) []string { if zone != nil { q = q.Equals("zone_id", zone.Id) } + q = q.IsNotEmpty("host_type").IsNotNull("host_type") q = q.Distinct() rows, err := q.Rows() if err != nil { @@ -58,6 +73,8 @@ func getStorageTypes(zone *SZone) []string { if zone != nil { q = q.Equals("zone_id", zone.Id) } + q = q.IsNotEmpty("storage_type").IsNotNull("storage_type") + q = q.IsNotEmpty("medium_type").IsNotNull("medium_type") q = q.Distinct() rows, err := q.Rows() if err != nil { diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 511573f3ab..6e38db1ae8 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -76,6 +76,8 @@ func init() { type SDisk struct { db.SSharableVirtualResourceBase + SBillingResourceBase + DiskFormat string `width:"32" charset:"ascii" nullable:"false" default:"qcow2" list:"user"` // Column(VARCHAR(32, charset='ascii'), nullable=False, default='qcow2') DiskSize int `nullable:"false" list:"user"` // Column(Integer, nullable=False) # in MB AccessPath string `width:"256" charset:"ascii" nullable:"true" get:"user"` // = Column(VARCHAR(256, charset='ascii'), nullable=True) @@ -650,6 +652,9 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke self.IsEmulated = extDisk.IsEmulated() + self.BillingType = extDisk.GetBillingType() + self.ExpiredAt = extDisk.GetExpiredAt() + self.ProjectId = userCred.GetProjectId() return nil @@ -693,6 +698,9 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl disk.IsEmulated = extDisk.IsEmulated() + disk.BillingType = extDisk.GetBillingType() + disk.ExpiredAt = extDisk.GetExpiredAt() + err := manager.TableSpec().Insert(&disk) if err != nil { log.Errorf("newFromCloudZone fail %s", err) @@ -855,27 +863,6 @@ func parseIsoInfo(ctx context.Context, userCred mcclient.TokenCredential, info s return image.Id, nil } -// def get_disk_spec_v2(conf): -// def _get_spec(storages): -// spec = {} -// for adapter, ss in group_by_adapter(storages).items(): -// if len(ss) == 0: -// continue -// spec[adapter] = get_disk_spec(ss) -// return spec - -// spec = {} -// for driver in DISK_DRIVERS: -// storages = [s for s in conf if s['driver'] == driver] -// if len(storages) != 0: -// spec[driver] = _get_spec(storages) -// return spec - -func GetDiskSpecV2(storageInfo jsonutils.JSONObject) *jsonutils.JSONDict { - // ToDo - return nil -} - func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { if len(diskConfig.ImageId) > 0 { self.TemplateId = diskConfig.ImageId @@ -897,6 +884,37 @@ func (self *SDisk) fetchDiskInfo(diskConfig *SDiskConfig) { self.DiskSize = diskConfig.Size } +type DiskInfo struct { + ImageId string + Fs string + MountPoint string + Format string + Size int64 + StorageId string + Backend string + MediumType string + Driver string + Cache string +} + +func (self *SDisk) ToDiskInfo() DiskInfo { + ret := DiskInfo{ + ImageId: self.GetTemplateId(), + Fs: self.GetFsFormat(), + MountPoint: self.GetMountPoint(), + Format: self.DiskFormat, + Size: int64(self.DiskSize), + } + storage := self.GetStorage() + if storage == nil { + return ret + } + ret.StorageId = storage.Id + ret.Backend = storage.StorageType + ret.MediumType = storage.MediumType + return ret +} + func (self *SDisk) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { // override log.Infof("disk delete do nothing") @@ -1050,6 +1068,8 @@ func (self *SDisk) GetShortDesc() *jsonutils.JSONDict { desc.Add(jsonutils.NewString(priceKey), "price_key") } + desc.Add(jsonutils.NewString(self.GetChargeType()), "charge_type") + if hypervisor := self.GetMetadata("hypervisor", nil); len(hypervisor) > 0 { desc.Add(jsonutils.NewString(hypervisor), "hypervisor") } @@ -1073,6 +1093,10 @@ func (self *SDisk) getDev() string { return self.GetMetadata("dev", nil) } +func (self *SDisk) GetMountPoint() string { + return self.GetMetadata("mountpoint", nil) +} + func (self *SDisk) isReady() bool { return self.Status == DISK_READY } diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index 3cb11e1e07..db60b1dbec 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -39,6 +39,8 @@ const ( EIP_STATUS_DISSOCIATE = "dissociate" EIP_STATUS_DISSOCIATE_FAIL = "dissociate_fail" + EIP_STATUS_CHANGE_BANDWIDTH = "change_bandwidth" + EIP_CHARGE_TYPE_BY_TRAFFIC = "traffic" EIP_CHARGE_TYPE_BY_BANDWIDTH = "bandwidth" EIP_CHARGE_TYPE_DEFAULT = EIP_CHARGE_TYPE_BY_TRAFFIC @@ -178,7 +180,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien } func (self *SElasticip) SyncInstanceWithCloudEip(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudEIP) error { - vm := self.getVM() + vm := self.GetAssociateVM() vmExtId := ext.GetAssociationExternalId() if vm == nil && len(vmExtId) == 0 { @@ -281,7 +283,7 @@ func (manager *SElasticipManager) getEipForInstance(instanceType string, instanc return &eip, nil } -func (self *SElasticip) getVM() *SGuest { +func (self *SElasticip) GetAssociateVM() *SGuest { if self.AssociateType == "server" && len(self.AssociateId) > 0 { return GuestManager.FetchGuestById(self.AssociateId) } @@ -292,7 +294,7 @@ func (self *SElasticip) Dissociate(ctx context.Context, userCred mcclient.TokenC if len(self.AssociateType) == 0 { return nil } - vm := self.getVM() + vm := self.GetAssociateVM() if vm == nil { log.Errorf("dissociate VM not exists???") } @@ -305,7 +307,9 @@ func (self *SElasticip) Dissociate(ctx context.Context, userCred mcclient.TokenC return err } if vm != nil { + db.OpsLog.LogDetachEvent(vm, self, userCred, self.GetShortDesc()) db.OpsLog.LogEvent(self, db.ACT_EIP_DETACH, vm.GetShortDesc(), userCred) + db.OpsLog.LogEvent(vm, db.ACT_EIP_DETACH, self.GetShortDesc(), userCred) } if self.Mode == EIP_MODE_INSTANCE_PUBLICIP { self.Delete(ctx, userCred) @@ -325,7 +329,11 @@ func (self *SElasticip) AssociateVM(userCred mcclient.TokenCredential, vm *SGues if err != nil { return err } + + db.OpsLog.LogAttachEvent(vm, self, userCred, self.GetShortDesc()) db.OpsLog.LogEvent(self, db.ACT_EIP_ATTACH, vm.GetShortDesc(), userCred) + db.OpsLog.LogEvent(vm, db.ACT_EIP_ATTACH, self.GetShortDesc(), userCred) + return nil } @@ -638,7 +646,7 @@ func (self *SElasticip) GetCustomizeColumns(ctx context.Context, userCred mcclie } func (self *SElasticip) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { - vm := self.getVM() + vm := self.GetAssociateVM() if vm != nil { extra.Add(jsonutils.NewString(vm.GetName()), "associate_name") } @@ -704,6 +712,9 @@ func (self *SElasticip) PerformChangeBandwidth(ctx context.Context, userCred mcc } func (self *SElasticip) StartEipChangeBandwidthTask(ctx context.Context, userCred mcclient.TokenCredential, bandwidth int64) error { + + self.SetStatus(userCred, EIP_STATUS_CHANGE_BANDWIDTH, "change bandwidth") + params := jsonutils.NewDict() params.Add(jsonutils.NewInt(bandwidth), "bandwidth") @@ -722,6 +733,8 @@ func (self *SElasticip) DoChangeBandwidth(userCred mcclient.TokenCredential, ban return nil }) + self.SetStatus(userCred, EIP_STATUS_READY, "finish change bandwidth") + if err != nil { log.Errorf("DoChangeBandwidth update fail %s", err) return err diff --git a/pkg/compute/models/guestdisks.go b/pkg/compute/models/guestdisks.go index 26e0c74c0c..484f906b86 100644 --- a/pkg/compute/models/guestdisks.go +++ b/pkg/compute/models/guestdisks.go @@ -211,3 +211,14 @@ func (self *SGuestdisk) Delete(ctx context.Context, userCred mcclient.TokenCrede func (self *SGuestdisk) Detach(ctx context.Context, userCred mcclient.TokenCredential) error { return db.DetachJoint(ctx, userCred, self) } + +func (self *SGuestdisk) ToDiskInfo() DiskInfo { + disk := self.GetDisk() + if disk == nil { + return DiskInfo{} + } + info := disk.ToDiskInfo() + info.Driver = self.Driver + info.Cache = self.CacheMode + return info +} diff --git a/pkg/compute/models/guestnetworks.go b/pkg/compute/models/guestnetworks.go index a080f4f3cd..2df92532e1 100644 --- a/pkg/compute/models/guestnetworks.go +++ b/pkg/compute/models/guestnetworks.go @@ -4,6 +4,7 @@ import ( "context" "crypto/md5" "crypto/rand" + "database/sql" "fmt" "io" "regexp" @@ -16,7 +17,6 @@ import ( "yunion.io/x/pkg/util/regutils" "yunion.io/x/sqlchemy" - "database/sql" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/compute/options" @@ -292,9 +292,11 @@ func (self *SGuestnetwork) ValidateUpdateData(ctx context.Context, userCred mccl if err != nil { return nil, fmt.Errorf("fail to fetch index %s", err) } - q := GuestnetworkManager.Query().Equals("guest_id", self.GuestId) - q = q.NotEquals("network_id", self.NetworkId).Equals("idnex", index) - if q.Count() > 0 { + q := GuestnetworkManager.Query().SubQuery() + count := q.Query().Filter(sqlchemy.Equals(q.Field("guest_id"), self.GuestId)). + Filter(sqlchemy.NotEquals(q.Field("network_id"), self.NetworkId)). + Filter(sqlchemy.Equals(q.Field("index"), index)).Count() + if count > 0 { return nil, fmt.Errorf("NIC Index %d has been occupied", index) } } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index d30e0e90aa..42285ef0ec 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -35,6 +35,8 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/onecloud/pkg/util/logclient" + "yunion.io/x/onecloud/pkg/util/seclib2" ) const ( @@ -73,9 +75,10 @@ const ( VM_MIGRATING = "migrating" VM_MIGRATE_FAILED = "migrate_failed" - VM_CHANGE_FLAVOR = "change_flavor" - VM_REBUILD_ROOT = "rebuild_root" - VM_REBUILD_ROOT_FAIL = "rebuild_root_fail" + VM_CHANGE_FLAVOR = "change_flavor" + VM_CHANGE_FLAVOR_FAIL = "change_flavor_fail" + VM_REBUILD_ROOT = "rebuild_root" + VM_REBUILD_ROOT_FAIL = "rebuild_root_fail" VM_START_SNAPSHOT = "snapshot_start" VM_SNAPSHOT = "snapshot" @@ -97,6 +100,9 @@ const ( VM_RESTORE_STATE = "restore_state" VM_RESTORE_FAILED = "restore_failed" + VM_ASSOCIATE_EIP = "associate_eip" + VM_DISSOCIATE_EIP = "dissociate_eip" + VM_REMOVE_STATEFILE = "remove_state" VM_ADMIN = "admin" @@ -155,6 +161,8 @@ func init() { type SGuest struct { db.SVirtualResourceBase + SBillingResourceBase + VcpuCount int8 `nullable:"false" default:"1" list:"user" create:"optional"` // Column(TINYINT, nullable=False, default=1) VmemSize int `nullable:"false" list:"user" create:"required"` // Column(Integer, nullable=False) @@ -271,8 +279,8 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ if count > 0 { sgq := guestdisks.Query(guestdisks.Field("guest_id")). Filter(sqlchemy.AND( - sqlchemy.Equals(guestdisks.Field("disk_id"), disk.Id), - sqlchemy.IsFalse(guestdisks.Field("deleted")))) + sqlchemy.Equals(guestdisks.Field("disk_id"), disk.Id), + sqlchemy.IsFalse(guestdisks.Field("deleted")))) q = q.Filter(sqlchemy.In(q.Field("id"), sgq)) } else { hosts := HostManager.Query().SubQuery() @@ -280,11 +288,11 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ storages := StorageManager.Query().SubQuery() sq := hosts.Query(hosts.Field("id")). Join(hoststorages, sqlchemy.AND( - sqlchemy.Equals(hoststorages.Field("host_id"), hosts.Field("id")), - sqlchemy.IsFalse(hoststorages.Field("deleted")))). + sqlchemy.Equals(hoststorages.Field("host_id"), hosts.Field("id")), + sqlchemy.IsFalse(hoststorages.Field("deleted")))). Join(storages, sqlchemy.AND( - sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")), - sqlchemy.IsFalse(storages.Field("deleted")))). + sqlchemy.Equals(storages.Field("id"), hoststorages.Field("storage_id")), + sqlchemy.IsFalse(storages.Field("deleted")))). Filter(sqlchemy.Equals(storages.Field("id"), disk.StorageId)).SubQuery() q = q.In("host_id", sq) } @@ -338,8 +346,8 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ isodev := IsolatedDeviceManager.Query().SubQuery() sgq := isodev.Query(isodev.Field("guest_id")). Filter(sqlchemy.AND( - sqlchemy.IsNotNull(isodev.Field("guest_id")), - sqlchemy.Startswith(isodev.Field("dev_type"), "GPU"))) + sqlchemy.IsNotNull(isodev.Field("guest_id")), + sqlchemy.Startswith(isodev.Field("dev_type"), "GPU"))) showGpu := utils.ToBool(gpu) cond := sqlchemy.NotIn if showGpu { @@ -516,6 +524,16 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To if err != nil { return nil, err } + + if vmemSize > 0 || vcpuCount > 0 { + if !utils.IsInStringArray(self.Status, []string{VM_READY}) { + return nil, httperrors.NewInvalidStatusError("Cannot modify Memory and CPU in status %s", self.Status) + } + if self.GetHypervisor() == HYPERVISOR_BAREMETAL { + return nil, httperrors.NewInputParameterError("Cannot modify memory for baremetal") + } + } + if vmemSize > 0 { data.Add(jsonutils.NewInt(int64(vmemSize)), "vmem_size") } @@ -533,14 +551,6 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To return nil, httperrors.NewInputParameterError("name is to short") } } - /* if self.GetHypervisor() == HYPERVISOR_BAREMETAL { - return nil, httperrors.NewInputParameterError("Cannot modify memory for baremetal") - } - if ! utils.IsInStringArray(self.Status, []string {VM_READY}) { - return nil, httperrors.NewInvalidStatusError("Cannot modify Memory and CPU in status %s", self.Status) - }*/ - // return nil, httperrors.NewInputParameterError("cannot update guest vmem_size") - //} return self.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) } @@ -558,6 +568,14 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m data.Add(jsonutils.NewInt(int64(vmemSize)), "vmem_size") data.Add(jsonutils.NewInt(int64(vcpuCount)), "vcpu_count") + resetPassword := jsonutils.QueryBoolean(data, "reset_password", true) + passwd, _ := data.GetString("password") + if resetPassword && len(passwd) > 0 { + if ! seclib2.MeetComplxity(passwd) { + return nil, httperrors.NewWeakPasswordError() + } + } + disk0Json, _ := data.Get("disk.0") if disk0Json == nil { return nil, httperrors.NewInputParameterError("No disk information provided") @@ -1302,6 +1320,9 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token self.IsEmulated = extVM.IsEmulated() + self.BillingType = extVM.GetBillingType() + self.ExpiredAt = extVM.GetExpiredAt() + return nil }) if err != nil { @@ -1349,6 +1370,9 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. guest.IsEmulated = extVM.IsEmulated() + guest.BillingType = extVM.GetBillingType() + guest.ExpiredAt = extVM.GetExpiredAt() + guest.HostId = host.Id guest.ProjectId = userCred.GetProjectId() @@ -2115,7 +2139,7 @@ func (self *SGuest) attachIsolatedDevice(userCred mcclient.TokenCredential, dev if dev.HostId != self.HostId { return fmt.Errorf("Isolated device and guest are not located in the same host") } - _, err := self.GetModelManager().TableSpec().Update(self, func() error { + _, err := IsolatedDeviceManager.TableSpec().Update(dev, func() error { dev.GuestId = self.Id return nil }) @@ -2188,6 +2212,9 @@ func (self *SGuest) CategorizeNics() SGuestNicCategory { func (self *SGuest) StartGuestDeployTask(ctx context.Context, userCred mcclient.TokenCredential, kwargs *jsonutils.JSONDict, action string, parentTaskId string) error { self.SetStatus(userCred, VM_START_DEPLOY, "") + if kwargs == nil { + kwargs = jsonutils.NewDict() + } kwargs.Add(jsonutils.NewString(action), "deploy_action") task, err := taskman.TaskManager.NewTask(ctx, "GuestDeployTask", self, userCred, kwargs, parentTaskId, "", nil) if err != nil { @@ -2402,41 +2429,81 @@ func (self *SGuest) PerformPurge(ctx context.Context, userCred mcclient.TokenCre return nil, err } +func (self *SGuest) setKeypairId(userCred mcclient.TokenCredential, keypairId string) error { + diff, err := self.GetModelManager().TableSpec().Update(self, func() error { + self.KeypairId = keypairId + return nil + }) + if err != nil { + db.OpsLog.LogEvent(self, db.ACT_UPDATE, diff, userCred) + } + return err +} + func (self *SGuest) AllowPerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return self.IsOwner(userCred) } func (self *SGuest) PerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { imageId, _ := data.GetString("image_id") - if utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_ADMIN}) { - if !data.Contains("image_id") { - gdc := self.CategorizeDisks() - imageId = gdc.Root.GetTemplateId() - if len(imageId) == 0 { - return nil, httperrors.NewBadRequestError("No template for root disk") - } - img, err := CachedimageManager.getImageInfo(ctx, userCred, imageId, false) - if err != nil { - return nil, httperrors.NewBadRequestError("Template %s not accessible: %s", imageId, err.Error()) - } - osType, _ := img.Properties["os_type"] - osName := self.GetMetadata("os_name", userCred) - if len(osName) == 0 && len(osType) == 0 && strings.ToLower(osType) != strings.ToLower(osName) { - return nil, httperrors.NewBadRequestError("Cannot switch OS between %s-%s", osName, osType) - } - } - autoStart := jsonutils.QueryBoolean(data, "auto_start", false) - var needStop = false - if self.Status == VM_RUNNING { - needStop = true - } - err := self.StartRebuildRootTask(ctx, userCred, imageId, needStop, autoStart) - return nil, err + if ! utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_ADMIN}) { + return nil, httperrors.NewInvalidStatusError("Cannot reset root in status %s", self.Status) } - return nil, httperrors.NewInvalidStatusError("Cannot reset root in status %s", self.Status) + + if !data.Contains("image_id") { + gdc := self.CategorizeDisks() + imageId = gdc.Root.GetTemplateId() + if len(imageId) == 0 { + return nil, httperrors.NewBadRequestError("No template for root disk") + } + img, err := CachedimageManager.getImageInfo(ctx, userCred, imageId, false) + if err != nil { + return nil, httperrors.NewBadRequestError("Template %s not accessible: %s", imageId, err.Error()) + } + osType, _ := img.Properties["os_type"] + osName := self.GetMetadata("os_name", userCred) + if len(osName) == 0 && len(osType) == 0 && strings.ToLower(osType) != strings.ToLower(osName) { + return nil, httperrors.NewBadRequestError("Cannot switch OS between %s-%s", osName, osType) + } + } + + autoStart := jsonutils.QueryBoolean(data, "auto_start", false) + var needStop = false + if self.Status == VM_RUNNING { + needStop = true + } + resetPasswd := jsonutils.QueryBoolean(data, "reset_password", true) + passwd, _ := data.GetString("password") + if len(passwd) > 0 { + if ! seclib2.MeetComplxity(passwd) { + return nil, httperrors.NewWeakPasswordError() + } + } + + keypairStr := jsonutils.GetAnyString(data, []string{"keypair", "keypair_id"}) + if len(keypairStr) > 0 { + keypairObj, err := KeypairManager.FetchByIdOrName(userCred.GetUserId(), keypairStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError("keypair %s not found", keypairStr) + } else { + return nil, httperrors.NewGeneralError(err) + } + } + if self.KeypairId != keypairObj.GetId() { + err = self.setKeypairId(userCred, keypairObj.GetId()) + if err != nil { + return nil, httperrors.NewGeneralError(err) + } + } + } + + err := self.StartRebuildRootTask(ctx, userCred, imageId, needStop, autoStart, passwd, resetPasswd) + return nil, err + } -func (self *SGuest) StartRebuildRootTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, needStop, autoStart bool) error { +func (self *SGuest) StartRebuildRootTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, needStop, autoStart bool, passwd string, resetPasswd bool) error { data := jsonutils.NewDict() data.Set("image_id", jsonutils.NewString(imageId)) if needStop { @@ -2445,6 +2512,14 @@ func (self *SGuest) StartRebuildRootTask(ctx context.Context, userCred mcclient. if autoStart { data.Set("auto_start", jsonutils.JSONTrue) } + if resetPasswd { + data.Set("reset_password", jsonutils.JSONTrue) + } else { + data.Set("reset_password", jsonutils.JSONFalse) + } + if len(passwd) > 0 { + data.Set("password", jsonutils.NewString(passwd)) + } if self.GetHypervisor() == HYPERVISOR_BAREMETAL { task, err := taskman.TaskManager.NewTask(ctx, "BaremetalServerRebuildRootTask", self, userCred, data, "", "", nil) if err != nil { @@ -2468,6 +2543,74 @@ func (self *SGuest) DetachDisk(ctx context.Context, disk *SDisk, userCred mcclie } } +func (self *SGuest) AllowPerformCreatedisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + var diskIdx, diskSize = 0, 0 + disksConf := jsonutils.NewDict() + diskSizes := make(map[string]int, 0) + diskSeq := fmt.Sprintf("disk.%d", diskIdx) + for data.Contains(diskSeq) { + diskDef, _ := data.Get(diskSeq) + diskInfo, err := parseDiskInfo(ctx, userCred, diskDef) + if err != nil { + logclient.AddActionLog(self, logclient.ACT_CREATE, err.Error(), userCred, false) + return nil, httperrors.NewBadRequestError(err.Error()) + } + disksConf.Set(diskSeq, jsonutils.Marshal(diskInfo)) + if _, ok := diskSizes[diskInfo.Backend]; !ok { + diskSizes[diskInfo.Backend] = diskInfo.Size + } else { + diskSizes[diskInfo.Backend] += diskInfo.Size + } + diskSize += diskInfo.Size + diskIdx += 1 + diskSeq = fmt.Sprintf("disk.%d", diskIdx) + } + if diskIdx == 0 { + logclient.AddActionLog(self, logclient.ACT_CREATE, "No Disk Info Provided", userCred, false) + return nil, httperrors.NewBadRequestError("No Disk Info Provided") + } + host := self.GetHost() + if host == nil { + logclient.AddActionLog(self, logclient.ACT_CREATE, "No valid host", userCred, false) + return nil, httperrors.NewBadRequestError("No valid host") + } + for backend, size := range diskSizes { + storage := host.GetLeastUsedStorage(backend) + if storage == nil { + logclient.AddActionLog(self, logclient.ACT_CREATE, "No valid storage on current host", userCred, false) + return nil, httperrors.NewBadRequestError("No valid storage on current host") + } + if storage.GetCapacity() < size { + logclient.AddActionLog(self, logclient.ACT_CREATE, "Not eough storage space on current host", userCred, false) + return nil, httperrors.NewBadRequestError("Not eough storage space on current host") + } + } + pendingUsage := &SQuota{ + Storage: diskSize, + } + err := QuotaManager.CheckSetPendingQuota(ctx, userCred, self.ProjectId, pendingUsage) + if err != nil { + logclient.AddActionLog(self, logclient.ACT_CREATE, err.Error(), userCred, false) + return nil, httperrors.NewBadRequestError(err.Error()) + } + + lockman.LockObject(ctx, host) + defer lockman.ReleaseObject(ctx, host) + + err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage) + if err != nil { + QuotaManager.CancelPendingUsage(ctx, userCred, self.ProjectId, nil, pendingUsage) + logclient.AddActionLog(self, logclient.ACT_CREATE, err.Error(), userCred, false) + return nil, httperrors.NewBadRequestError(err.Error()) + } + err = self.StartGuestCreateDiskTask(ctx, userCred, disksConf, "") + return nil, err +} + func (self *SGuest) AllowPerformDetachdisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return self.IsOwner(userCred) } @@ -2514,6 +2657,196 @@ func (self *SGuest) PerformDetachdisk(ctx context.Context, userCred mcclient.Tok return nil, httperrors.NewResourceNotFoundError("Disk %s not found", diskId) } +func (self *SGuest) AllowPerformDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Hypervisor != HYPERVISOR_KVM { + return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) + } + if self.Status != VM_READY { + msg := "Only allowed to attach isolated device when guest is ready" + logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewInvalidStatusError(msg) + } + device, err := data.GetString("device") + if err != nil { + msg := "Missing isolated device" + logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError(msg) + } + iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred.GetProjectId(), device) + if err != nil { + msg := fmt.Sprintf("Isolated device %s not found", device) + logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError(msg) + } + dev := iDev.(*SIsolatedDevice) + host := self.GetHost() + lockman.LockObject(ctx, host) + defer lockman.ReleaseObject(ctx, host) + err = self.detachIsolateDevice(userCred, dev) + return nil, err +} + +func (self *SGuest) detachIsolateDevice(userCred mcclient.TokenCredential, dev *SIsolatedDevice) error { + if dev.GuestId != self.Id { + msg := "Isolated device is not attached to this guest" + logclient.AddActionLog(self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) + return httperrors.NewBadRequestError(msg) + } + _, err := self.GetModelManager().TableSpec().Update(dev, func() error { + dev.GuestId = "" + return nil + }) + if err != nil { + return err + } + db.OpsLog.LogEvent(self, db.ACT_GUEST_DETACH_ISOLATED_DEVICE, dev.GetShortDesc(), userCred) + return nil +} + +func (self *SGuest) AllowPerformAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return userCred.IsSystemAdmin() +} + +func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Hypervisor != HYPERVISOR_KVM { + return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) + } + if self.Status != VM_READY { + msg := "Only allowed to attach isolated device when guest is ready" + logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewInvalidStatusError(msg) + } + device, err := data.GetString("device") + if err != nil { + msg := "Missing isolated device" + logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError(msg) + } + iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred.GetProjectId(), device) + if err != nil { + msg := fmt.Sprintf("Isolated device %s not found", device) + logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false) + return nil, httperrors.NewBadRequestError(msg) + } + dev := iDev.(*SIsolatedDevice) + host := self.GetHost() + lockman.LockObject(ctx, host) + defer lockman.ReleaseObject(ctx, host) + err = self.attachIsolatedDevice(userCred, dev) + var msg string + if err != nil { + msg = err.Error() + } + logclient.AddActionLog(self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, err == nil) + return nil, err +} + +func (self *SGuest) AllowPerformDetachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Status != VM_READY { + return nil, httperrors.NewInvalidStatusError("Cannot detach network in status %s", self.Status) + } + reserve := jsonutils.QueryBoolean(data, "reserve", false) + netId, err := data.GetString("net_id") + if err != nil { + return nil, httperrors.NewBadRequestError(err.Error()) + } + iNetwork, err := NetworkManager.FetchById(netId) + if err != nil { + return nil, httperrors.NewNotFoundError("Network %s not found", netId) + } + network := iNetwork.(*SNetwork) + err = self.detachNetwork(ctx, userCred, network, reserve, true) + return nil, err +} + +func (self *SGuest) AllowPerformAttachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Status == VM_READY { + // owner_cred = self.get_owner_user_cred() >.< + netDesc, err := data.Get("net_desc") + if err != nil { + return nil, httperrors.NewBadRequestError(err.Error()) + } + conf, err := parseNetworkInfo(userCred, netDesc) + if err != nil { + return nil, err + } + err = isValidNetworkInfo(userCred, conf) + if err != nil { + return nil, httperrors.NewBadRequestError(err.Error()) + } + var inicCnt, enicCnt, ibw, ebw int + if isExitNetworkInfo(conf) { + enicCnt = 1 + ebw = conf.BwLimit + } else { + inicCnt = 1 + ibw = conf.BwLimit + } + pendingUsage := &SQuota{ + Port: inicCnt, + Eport: enicCnt, + Bw: ibw, + Ebw: ebw, + } + projectId := self.GetOwnerProjectId() + err = QuotaManager.CheckSetPendingQuota(ctx, userCred, projectId, pendingUsage) + if err != nil { + return nil, httperrors.NewOutOfQuotaError(err.Error()) + } + host := self.GetHost() + err = self.attach2NetworkDesc(ctx, userCred, host, conf, pendingUsage) + if err != nil { + QuotaManager.CancelPendingUsage(ctx, userCred, projectId, nil, pendingUsage) + return nil, httperrors.NewBadRequestError(err.Error()) + } + host.ClearSchedDescCache() + err = self.StartGuestDeployTask(ctx, userCred, nil, "deploy", "") + return nil, err + } + return nil, httperrors.NewBadRequestError("Cannot attach network in status %s", self.Status) +} + +func (self *SGuest) AllowPerformChangeBandwidth(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) || userCred.IsSystemAdmin() +} + +func (self *SGuest) PerformChangeBandwidth(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING}) { + guestnics := self.GetNetworks() + index, err := data.Int("index") + if err != nil || index > int64(len(guestnics)) { + return nil, httperrors.NewBadRequestError("Index Not fount or out of NIC index") + } + bandwidth, err := data.Int("bandwidth") + if err != nil || bandwidth <= 0 { + return nil, httperrors.NewBadRequestError("Bandwidth must be larger than 0") + } + guestnic := &guestnics[index] + if guestnic.BwLimit != int(bandwidth) { + GuestnetworkManager.TableSpec().Update(guestnic, func() error { + guestnic.BwLimit = int(bandwidth) + return nil + }) + err := self.StartSyncTask(ctx, userCred, false, "") + return nil, err + } + return nil, nil + } + return nil, httperrors.NewBadRequestError("Cannot change bandwidth in status %s", self.Status) +} + func (self *SGuest) AllowPerformChangeConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { return self.IsOwner(userCred) || self.IsAdmin(userCred) } @@ -2608,12 +2941,12 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T } provider, e := self.GetHost().GetDriver() - if e != nil { + /*if e != nil { log.Errorf("Get Provider Error: %s", e) return nil, httperrors.NewInsufficientResourceError("Provider Not Found") - } + }*/ - if !provider.IsPublicCloud() { + if e != nil || !provider.IsPublicCloud() { for storageId, needSize := range diskSizes { iStorage, err := StorageManager.FetchById(storageId) if err != nil { @@ -2637,6 +2970,9 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T if jsonutils.QueryBoolean(data, "auto_start", false) { confs.Add(jsonutils.NewBool(true), "auto_start") } + + log.Debugf("%s", confs.String()) + pendingUsage := &SQuota{} if addCpu > 0 { pendingUsage.Cpu = addCpu @@ -2873,17 +3209,23 @@ func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, host *SHost, para deployAction = "deploy" } - resetPasswd := true - if deployAction == "deploy" { - resetPasswd = jsonutils.QueryBoolean(params, "reset_password", false) - } + // resetPasswd := true + // if deployAction == "deploy" { + resetPasswd := jsonutils.QueryBoolean(params, "reset_password", true) + //} if resetPasswd { config.Add(jsonutils.JSONTrue, "reset_password") + passwd, _ := params.GetString("password") + if len(passwd) > 0 { + config.Add(jsonutils.NewString(passwd), "password") + } keypair := self.getKeypair() if keypair != nil { config.Add(jsonutils.NewString(keypair.PublicKey), "public_key") } + } else { + config.Add(jsonutils.JSONFalse, "reset_password") } config.Add(jsonutils.NewString(deployAction), "action") @@ -3203,7 +3545,7 @@ func (manager *SGuestManager) FetchGuestById(guestId string) *SGuest { func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { if checkStatus { - if !utils.IsInStringArray(self.Status, []string{VM_SCHEDULE_FAILED}) { + if utils.IsInStringArray(self.Status, []string{VM_SCHEDULE_FAILED}) { return nil } } @@ -3215,12 +3557,11 @@ func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { guestdisks := self.GetDisks() diskSpecs := jsonutils.NewArray() for _, guestdisk := range guestdisks { - disk := guestdisk.GetDisk() + info := guestdisk.ToDiskInfo() diskSpec := jsonutils.NewDict() - diskSpec.Set("size", jsonutils.NewInt(int64(disk.DiskSize))) - s := disk.GetStorage() - diskSpec.Set("backend", jsonutils.NewString(s.StorageType)) - diskSpec.Set("medium_type", jsonutils.NewString(s.MediumType)) + diskSpec.Set("size", jsonutils.NewInt(info.Size)) + diskSpec.Set("backend", jsonutils.NewString(info.Backend)) + diskSpec.Set("medium_type", jsonutils.NewString(info.MediumType)) diskSpecs.Add(diskSpec) } spec.Set("disk", diskSpecs) @@ -3255,6 +3596,63 @@ func (self *SGuest) GetSpec(checkStatus bool) *jsonutils.JSONDict { return spec } +func (manager *SGuestManager) GetSpecIdent(spec *jsonutils.JSONDict) []string { + cpuCount, _ := spec.Int("cpu") + memSize, _ := spec.Int("mem") + memSizeMB, _ := utils.GetSizeMB(fmt.Sprintf("%d", memSize), "M") + specKeys := []string{ + fmt.Sprintf("cpu:%d", cpuCount), + fmt.Sprintf("mem:%dM", memSizeMB), + } + + countKey := func(kf func(*jsonutils.JSONDict) string, dataArray jsonutils.JSONObject) map[string]int64 { + countMap := make(map[string]int64) + datas, _ := dataArray.GetArray() + for _, data := range datas { + key := kf(data.(*jsonutils.JSONDict)) + if count, ok := countMap[key]; !ok { + countMap[key] = 1 + } else { + count++ + countMap[key] = count + } + } + return countMap + } + + kfuncs := map[string]func(*jsonutils.JSONDict) string{ + "disk": func(data *jsonutils.JSONDict) string { + backend, _ := data.GetString("backend") + mediumType, _ := data.GetString("medium_type") + size, _ := data.Int("size") + sizeGB, _ := utils.GetSizeGB(fmt.Sprintf("%d", size), "M") + return fmt.Sprintf("disk:%s_%s_%dG", backend, mediumType, sizeGB) + }, + "nic": func(data *jsonutils.JSONDict) string { + typ, _ := data.GetString("type") + bw, _ := data.Int("bandwidth") + return fmt.Sprintf("nic:%s_%dM", typ, bw) + }, + "gpu": func(data *jsonutils.JSONDict) string { + vendor, _ := data.GetString("vendor") + model, _ := data.GetString("model") + return fmt.Sprintf("gpu:%s_%s", vendor, model) + }, + } + + for sKey, kf := range kfuncs { + sArrary, err := spec.Get(sKey) + if err != nil { + log.Errorf("Get key %s array error: %v", sKey, err) + continue + } + for key, count := range countKey(kf, sArrary) { + specKeys = append(specKeys, fmt.Sprintf("%sx%d", key, count)) + } + } + return specKeys +} + func (self *SGuest) GetTemplateId() string { guestdisks := self.GetDisks() for _, guestdisk := range guestdisks { @@ -3290,6 +3688,8 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { desc.Add(jsonutils.NewString(priceKey), "price_key") } + desc.Add(jsonutils.NewString(self.GetChargeType()), "charge_type") + if len(self.ExternalId) > 0 { desc.Add(jsonutils.NewString(self.ExternalId), "externalId") } @@ -3300,7 +3700,7 @@ func (self *SGuest) GetShortDesc() *jsonutils.JSONDict { host := self.GetHost() if host != nil { hostSpec := host.GetSpec(false) - hostSpecIdent := host.GetSpecIdent(hostSpec) + hostSpecIdent := HostManager.GetSpecIdent(hostSpec) spec.Set("host_spec", jsonutils.NewString(strings.Join(hostSpecIdent, "/"))) } } @@ -3554,6 +3954,78 @@ func (self *SGuest) GetDetailsMonitor(ctx context.Context, userCred mcclient.Tok return nil, httperrors.NewInvalidStatusError("Cannot send command in status %s", self.Status) } +func (self *SGuest) AllowGetDetailsDesc(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) GetDetailsDesc(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + host := self.GetHost() + if host == nil { + return nil, httperrors.NewInvalidStatusError("No host for server") + } + desc := self.GetDriver().GetJsonDescAtHost(ctx, self, host) + return desc, nil +} + +func (self *SGuest) AllowPerformSendkeys(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformSendkeys(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + if self.Hypervisor != HYPERVISOR_KVM { + return nil, httperrors.NewUnsupportOperationError("Not allow for hypervisor %s", self.Hypervisor) + } + if self.Status != VM_RUNNING { + return nil, httperrors.NewInvalidStatusError("Cannot send keys in status %s", self.Status) + } + keys, err := data.GetString("keys") + if err != nil { + return nil, err + } + err = self.VerifySendKeys(keys) + if err != nil { + return nil, httperrors.NewBadRequestError(err.Error()) + } + cmd := fmt.Sprintf("sendkey %s", keys) + duration, err := data.Int("duration") + if err == nil { + cmd = fmt.Sprintf("%s %d", cmd, duration) + } + _, err = self.SendMonitorCommand(ctx, userCred, cmd) + return nil, err +} + +func (self *SGuest) VerifySendKeys(keyStr string) error { + keys := strings.Split(keyStr, "-") + for _, key := range keys { + if !self.IsLegalKey(key) { + return fmt.Errorf("Unknown key '%s'", key) + } + } + return nil +} + +func (self *SGuest) IsLegalKey(key string) bool { + singleKeys := "1234567890abcdefghijklmnopqrstuvwxyz" + legalKeys := []string{"ctrl", "ctrl_r", "alt", "alt_r", "shift", "shift_r", + "delete", "esc", "insert", "print", "spc", + "f1", "f2", "f3", "f4", "f5", "f6", + "f7", "f8", "f9", "f10", "f11", "f12", + "home", "pgup", "pgdn", "end", + "up", "down", "left", "right", + "tab", "minus", "equal", "backspace", "backslash", + "bracket_left", "bracket_right", "backslash", + "semicolon", "apostrophe", "grave_accent", "ret", + "comma", "dot", "slash", + "caps_lock", "num_lock", "scroll_lock"} + if len(key) > 1 && !utils.IsInStringArray(key, legalKeys) { + return false + } else if len(key) == 1 && !strings.Contains(singleKeys, key) { + return false + } + return true +} + func (self *SGuest) SendMonitorCommand(ctx context.Context, userCred mcclient.TokenCredential, cmd string) (jsonutils.JSONObject, error) { host := self.GetHost() url := fmt.Sprintf("%s/servers/%s/monitor", host.ManagerUri, self.Id) @@ -3588,7 +4060,7 @@ func (manager *SGuestManager) GetIpInProjectWithName(projectId, name string, isE sqlchemy.IsFalse(guests.Field("pending_deleted"))), sqlchemy.IsFalse(guests.Field("deleted")))). Join(networks, sqlchemy.AND(sqlchemy.Equals(networks.Field("id"), guestnics.Field("network_id")), - sqlchemy.IsFalse(networks.Field("deleted")))). + sqlchemy.IsFalse(networks.Field("deleted")))). Filter(sqlchemy.Equals(guests.Field("name"), name)). Filter(sqlchemy.NotEquals(guestnics.Field("ip_addr"), "")). Filter(sqlchemy.IsNotNull(guestnics.Field("ip_addr"))). @@ -3787,7 +4259,7 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T return nil, httperrors.NewUnsupportOperationError("fixed eip cannot be associated") } - eipVm := eip.getVM() + eipVm := eip.GetAssociateVM() if eipVm != nil { return nil, httperrors.NewConflictError("eip has been associated") } @@ -3805,6 +4277,8 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T return nil, httperrors.NewInputParameterError("cannot associate eip and instance in different provider") } + self.SetStatus(userCred, VM_ASSOCIATE_EIP, "associate eip") + params := jsonutils.NewDict() params.Add(jsonutils.NewString(self.ExternalId), "instance_external_id") params.Add(jsonutils.NewString(self.Id), "instance_id") @@ -3828,6 +4302,9 @@ func (self *SGuest) PerformDissociateEip(ctx context.Context, userCred mcclient. if eip == nil { return nil, httperrors.NewInvalidStatusError("No eip to dissociate") } + + self.SetStatus(userCred, VM_DISSOCIATE_EIP, "associate eip") + err = eip.StartEipDissociateTask(ctx, userCred, "") if err != nil { log.Errorf("fail to start dissociate task %s", err) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 01d418ee54..38c125c7c8 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "sort" "strconv" "yunion.io/x/jsonutils" @@ -19,6 +20,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/compute/options" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -521,16 +523,14 @@ func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { } spec := self.GetHardwareSpecification() spec.Remove("storage_info") - netInfo := jsonutils.NewArray() nifs := self.GetNetInterfaces() + var nicCount int64 for _, nif := range nifs { - netDesc := nif.getBaremetalJsonDesc() - nicType, err := netDesc.GetString("nic_type") - if err != nil && nicType != NIC_TYPE_IPMI { - netInfo.Add(netDesc) + if nif.NicType != NIC_TYPE_IPMI { + nicCount++ } } - spec.Set("nic_count", jsonutils.NewInt(int64(netInfo.Length()))) + spec.Set("nic_count", jsonutils.NewInt(nicCount)) manufacture, err := self.SysInfo.Get("manufacture") if err != nil { manufacture = jsonutils.NewString("Unknown") @@ -541,12 +541,73 @@ func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { model = jsonutils.NewString("Unknown") } spec.Set("model", model) - return nil + return spec } -func (self *SHost) GetSpecIdent(spec *jsonutils.JSONDict) []string { - // Todo - return []string{} +func (manager *SHostManager) GetSpecIdent(spec *jsonutils.JSONDict) []string { + nCpu, _ := spec.Int("cpu") + memSize, _ := spec.Int("mem") + memGB, err := utils.GetSizeGB(fmt.Sprintf("%d", memSize), "M") + if err != nil { + log.Errorf("Get mem size %d GB error: %v", memSize, err) + } + nicCount, _ := spec.Int("nic_count") + manufacture, _ := spec.GetString("manufacture") + model, _ := spec.GetString("model") + + specKeys := []string{ + fmt.Sprintf("cpu:%d", nCpu), + fmt.Sprintf("mem:%dG", memGB), + fmt.Sprintf("nic:%d", nicCount), + fmt.Sprintf("manufacture:%s", manufacture), + fmt.Sprintf("model:%s", model), + } + diskSpec, _ := spec.Get("disk") + if diskSpec != nil { + driverSpecs, _ := diskSpec.GetMap() + for driver, driverSpec := range driverSpecs { + specKeys = append(specKeys, parseDiskDriverSpec(driver, driverSpec)...) + } + } + sort.Strings(specKeys) + return specKeys +} + +func parseDiskDriverSpec(driver string, spec jsonutils.JSONObject) []string { + ret := make([]string, 0) + adapterSpecs, _ := spec.GetMap() + for adapterKey, adapterSpec := range adapterSpecs { + for _, diskType := range []string{baremetal.HDD_DISK_SPEC_TYPE, baremetal.SSD_DISK_SPEC_TYPE} { + sizeCountMap, _ := adapterSpec.GetMap(diskType) + if sizeCountMap == nil { + continue + } + for size, count := range sizeCountMap { + sizeGB, _ := utils.GetSizeGB(size, "M") + diskKey := fmt.Sprintf("disk:%s_%s_%s_%dGx%s", driver, adapterKey, diskType, sizeGB, count) + ret = append(ret, diskKey) + } + } + } + return ret +} + +func GetDiskSpecV2(storageInfo jsonutils.JSONObject) jsonutils.JSONObject { + storages := []baremetal.BaremetalStorage{} + err := storageInfo.Unmarshal(&storages) + if err != nil { + log.Errorf("Unmarshal to baremetal storage error: %v", err) + return nil + } + refStorages := func() []*baremetal.BaremetalStorage { + ret := make([]*baremetal.BaremetalStorage, len(storages)) + for i, s := range storages { + ret[i] = &s + } + return ret + }() + diskSpec := baremetal.GetDiskSpecV2(refStorages) + return jsonutils.Marshal(diskSpec) } func (self *SHost) GetHardwareSpecification() *jsonutils.JSONDict { diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index d88332cf5c..077518d6f1 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -113,6 +113,16 @@ func (manager *SIsolatedDeviceManager) ListItemFilter(ctx context.Context, q *sq if jsonutils.QueryBoolean(query, "unused", false) { q = q.IsEmpty("guest_id") } + zoneStr := jsonutils.GetAnyString(query, []string{"zone", "zone_id"}) + if len(zoneStr) > 0 { + zone, _ := ZoneManager.FetchByIdOrName("", zoneStr) + if zone == nil { + return nil, httperrors.NewResourceNotFoundError("Zone %s not found", zoneStr) + } + hosts := HostManager.Query().SubQuery() + sq := hosts.Query(hosts.Field("id")).Filter(sqlchemy.Equals(hosts.Field("zone_id"), zone.GetId())) + q = q.Filter(sqlchemy.In(q.Field("host_id"), sq)) + } return q, nil } @@ -307,6 +317,36 @@ func (manager *SIsolatedDeviceManager) findUnusedQuery() *sqlchemy.SQuery { return q } +func (manager *SIsolatedDeviceManager) UnusedGpuQuery() *sqlchemy.SQuery { + q := manager.findUnusedQuery() + q = q.Filter(sqlchemy.OR( + sqlchemy.Equals(q.Field("dev_type"), GPU_HPC_TYPE), + sqlchemy.Equals(q.Field("dev_type"), GPU_VGA_TYPE))) + return q +} + +func (manager *SIsolatedDeviceManager) FindUnusedByModels(models []string) ([]SIsolatedDevice, error) { + devs := make([]SIsolatedDevice, 0) + q := manager.findUnusedQuery() + q = q.In("model", models) + err := q.All(&devs) + if err != nil { + return nil, err + } + return devs, nil +} + +func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SIsolatedDevice, error) { + devs := make([]SIsolatedDevice, 0) + q := manager.UnusedGpuQuery() + q = q.Equals("host_id", hostId) + err := q.All(&devs) + if err != nil { + return nil, err + } + return devs, nil +} + func (manager *SIsolatedDeviceManager) findHostUnusedByModel(model string, hostId string) ([]SIsolatedDevice, error) { devs := make([]SIsolatedDevice, 0) q := manager.findUnusedQuery() @@ -381,8 +421,8 @@ func (self *SIsolatedDevice) getDesc() *jsonutils.JSONDict { return desc } -func (self *SIsolatedDevice) GetSpec(checkStatus bool) *jsonutils.JSONDict { - if checkStatus { +func (self *SIsolatedDevice) GetSpec(statusCheck bool) *jsonutils.JSONDict { + if statusCheck { if len(self.GuestId) > 0 { return nil } @@ -399,6 +439,19 @@ func (self *SIsolatedDevice) GetSpec(checkStatus bool) *jsonutils.JSONDict { return spec } +func (man *SIsolatedDeviceManager) GetSpecIdent(spec *jsonutils.JSONDict) []string { + log.Errorf("gpu get specIdent: %s", spec) + devType, _ := spec.GetString("dev_type") + vendor, _ := spec.GetString("vendor") + model, _ := spec.GetString("model") + keys := []string{ + fmt.Sprintf("type:%s", devType), + fmt.Sprintf("vendor:%s", vendor), + fmt.Sprintf("model:%s", model), + } + return keys +} + func (self *SIsolatedDevice) GetShortDesc() *jsonutils.JSONDict { desc := self.getDesc() desc.Add(jsonutils.NewString(self.Keyword()), "res_name") @@ -452,10 +505,40 @@ func (self *SIsolatedDevice) GetCustomizeColumns(ctx context.Context, userCred m return extra } +func (self *SIsolatedDevice) ClearSchedDescCache() error { + if len(self.HostId) == 0 { + return nil + } + host := self.getHost() + return host.ClearSchedDescCache() +} + +func (self *SIsolatedDevice) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + err := self.SStandaloneResourceBase.Delete(ctx, userCred) + if err != nil { + return err + } + return self.ClearSchedDescCache() +} + func (self *SIsolatedDevice) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { + if len(self.GuestId) > 0 { + if !jsonutils.QueryBoolean(data, "purge", false) { + return httperrors.NewBadRequestError("Isolated device used by server: %s", self.GuestId) + } + iGuest, err := GuestManager.FetchById(self.GuestId) + if err != nil { + return err + } + guest := iGuest.(*SGuest) + err = guest.detachIsolateDevice(userCred, self) + if err != nil { + return err + } + } host := self.getHost() if host != nil { db.OpsLog.LogEvent(host, db.ACT_HOST_DETACH_ISOLATED_DEVICE, self.GetShortDesc(), userCred) } - return nil + return self.RealDelete(ctx, userCred) } diff --git a/pkg/compute/models/netinterfaces.go b/pkg/compute/models/netinterfaces.go index 5275af03b6..6562deeae6 100644 --- a/pkg/compute/models/netinterfaces.go +++ b/pkg/compute/models/netinterfaces.go @@ -162,6 +162,9 @@ func (self *SNetInterface) getServerJsonDesc() *jsonutils.JSONDict { func (self *SNetInterface) getBaremetalJsonDesc() *jsonutils.JSONDict { bn := self.GetBaremetalNetwork() + if bn == nil { + return nil + } return self.toJson(bn.IpAddr, bn.GetNetwork()) } diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 33a529f53a..d090098234 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -734,6 +734,30 @@ func isExitNetworkInfo(netConfig *SNetworkConfig) bool { return false } +func (self *SNetwork) getZone() *SZone { + wire := self.GetWire() + if wire != nil { + return wire.GetZone() + } + return nil +} + +func (self *SNetwork) getVpc() *SVpc { + wire := self.GetWire() + if wire != nil { + return wire.getVpc() + } + return nil +} + +func (self *SNetwork) getRegion() *SCloudregion { + wire := self.GetWire() + if wire != nil { + return wire.getRegion() + } + return nil +} + func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { wire := self.GetWire() extra.Add(jsonutils.NewString(wire.Name), "wire") @@ -747,6 +771,34 @@ func (self *SNetwork) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSOND extra.Add(jsonutils.NewInt(int64(self.GetBaremetalNicsCount())), "bm_vnics") extra.Add(jsonutils.NewInt(int64(self.GetGroupNicsCount())), "group_vnics") extra.Add(jsonutils.NewInt(int64(self.GetReservedNicsCount())), "reserve_vnics") + + zone := self.getZone() + if zone != nil { + extra.Add(jsonutils.NewString(zone.GetId()), "zone_id") + extra.Add(jsonutils.NewString(zone.GetName()), "zone") + if len(zone.GetExternalId()) > 0 { + extra.Add(jsonutils.NewString(zone.GetExternalId()), "zone_external_id") + } + } + + region := self.getRegion() + if region != nil { + extra.Add(jsonutils.NewString(region.GetId()), "region_id") + extra.Add(jsonutils.NewString(region.GetName()), "region") + if len(region.GetExternalId()) > 0 { + extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id") + } + } + + vpc := self.getVpc() + if vpc != nil { + extra.Add(jsonutils.NewString(vpc.GetId()), "vpc_id") + extra.Add(jsonutils.NewString(vpc.GetName()), "vpc") + if len(vpc.GetExternalId()) > 0 { + extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id") + } + } + return extra } diff --git a/pkg/compute/models/specs.go b/pkg/compute/models/specs.go new file mode 100644 index 0000000000..841d1c87f8 --- /dev/null +++ b/pkg/compute/models/specs.go @@ -0,0 +1,115 @@ +package models + +import ( + "context" + "reflect" + "sort" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type ISpecModelManager interface { + db.IStandaloneModelManager + GetSpecIdent(spec *jsonutils.JSONDict) []string +} + +type ISpecModel interface { + db.IStandaloneModel + GetSpec(statusCheck bool) *jsonutils.JSONDict +} + +func GetAllModelSpecs(ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (jsonutils.JSONObject, error) { + mans := []ISpecModelManager{HostManager, IsolatedDeviceManager, GuestManager} + return GetModelsSpecs(ctx, userCred, query, mans...) +} + +func GetModelsSpecs(ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict, managers ...ISpecModelManager) (jsonutils.JSONObject, error) { + ret := jsonutils.NewDict() + for _, man := range managers { + spec, err := getModelSpecs(man, ctx, userCred, query) + if err != nil { + return nil, err + } + ret.Add(spec, man.KeywordPlural()) + } + return ret, nil +} + +func GetHostSpecs(ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (jsonutils.JSONObject, error) { + return getModelSpecs(HostManager, ctx, userCred, query) +} + +func GetIsolatedDeviceSpecs(ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (jsonutils.JSONObject, error) { + return getModelSpecs(IsolatedDeviceManager, ctx, userCred, query) +} + +func GetServerSpecs(ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (jsonutils.JSONObject, error) { + return getModelSpecs(GuestManager, ctx, userCred, query) +} + +func getModelSpecs(manager ISpecModelManager, ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (jsonutils.JSONObject, error) { + items, err := ListItems(manager, ctx, userCred, query) + if err != nil { + return nil, err + } + retDict := jsonutils.NewDict() + for _, obj := range items { + specObj := obj.(ISpecModel) + spec := specObj.GetSpec(true) + if spec == nil { + continue + } + specKeys := manager.GetSpecIdent(spec) + sort.Strings(specKeys) + specKey := strings.Join(specKeys, "/") + if oldSpec, _ := retDict.Get(specKey); oldSpec == nil { + spec.Add(jsonutils.NewInt(1), "count") + retDict.Add(spec, specKey) + } else { + count, _ := oldSpec.Int("count") + oldSpec.(*jsonutils.JSONDict).Set("count", jsonutils.NewInt(count+1)) + retDict.Set(specKey, oldSpec) + } + } + return retDict, nil +} + +func ListItems(manager db.IModelManager, ctx context.Context, userCred mcclient.TokenCredential, queryDict *jsonutils.JSONDict) ([]ISpecModel, error) { + q := manager.Query() + queryDict, err := manager.ValidateListConditions(ctx, userCred, queryDict) + if err != nil { + return nil, err + } + q, err = db.ListItemQueryFilters(manager, ctx, q, userCred, queryDict) + if err != nil { + return nil, err + } + rows, err := q.Rows() + if err != nil { + return nil, err + } + items := make([]ISpecModel, 0) + for rows.Next() { + item, err := db.NewModelObject(manager) + if err != nil { + return nil, err + } + itemInitValue := reflect.Indirect(reflect.ValueOf(item)) + item, err = db.NewModelObject(manager) + if err != nil { + return nil, err + } + itemValue := reflect.Indirect(reflect.ValueOf(item)) + itemValue.Set(itemInitValue) + err = q.Row2Struct(rows, item) + if err != nil { + return nil, err + } + items = append(items, item.(ISpecModel)) + } + return items, err +} diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index e2d11f3b72..67fd0cbc82 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -124,7 +124,12 @@ func (self *SVpc) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict extra.Add(jsonutils.NewInt(int64(self.GetWireCount())), "wire_count") extra.Add(jsonutils.NewInt(int64(self.GetNetworkCount())), "network_count") region := self.GetRegion() - extra.Add(jsonutils.NewString(region.GetName()), "region") + if region != nil { + extra.Add(jsonutils.NewString(region.GetName()), "region") + if len(region.GetExternalId()) > 0 { + extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id") + } + } return extra } diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 7dea974aed..345de3fd12 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -561,3 +561,48 @@ func (manager *SWireManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu return q, err } + +func (self *SWire) getRegion() *SCloudregion { + zone := self.GetZone() + if zone != nil { + return zone.GetRegion() + } + return nil +} + +func (self *SWire) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SStandaloneResourceBase.GetCustomizeColumns(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SWire) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { + extra := self.SStandaloneResourceBase.GetExtraDetails(ctx, userCred, query) + return self.getMoreDetails(extra) +} + +func (self *SWire) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict { + extra.Add(jsonutils.NewInt(int64(self.NetworkCount())), "networks") + zone := self.GetZone() + if zone != nil { + extra.Add(jsonutils.NewString(zone.GetName()), "zone") + if len(zone.GetExternalId()) > 0 { + extra.Add(jsonutils.NewString(zone.GetExternalId()), "zone_external_id") + } + } + region := self.getRegion() + if region != nil { + extra.Add(jsonutils.NewString(region.GetId()), "region_id") + extra.Add(jsonutils.NewString(region.GetName()), "region") + if len(region.GetExternalId()) > 0 { + extra.Add(jsonutils.NewString(region.GetExternalId()), "region_external_id") + } + } + vpc := self.getVpc() + if vpc != nil { + extra.Add(jsonutils.NewString(vpc.GetName()), "vpc") + if len(vpc.GetExternalId()) > 0 { + extra.Add(jsonutils.NewString(vpc.GetExternalId()), "vpc_external_id") + } + } + return extra +} \ No newline at end of file diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index f2c82736c7..8b1cc8c57e 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -479,7 +479,10 @@ func (self *SZone) AllowGetDetailsCapability(ctx context.Context, userCred mccli } func (self *SZone) GetDetailsCapability(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - capa := GetCapabilities(self) + capa, err := GetCapabilities(ctx, userCred, query, self) + if err != nil { + return nil, err + } return jsonutils.Marshal(&capa), nil } diff --git a/pkg/compute/specs/handler.go b/pkg/compute/specs/handler.go new file mode 100644 index 0000000000..2cf3b80d39 --- /dev/null +++ b/pkg/compute/specs/handler.go @@ -0,0 +1,223 @@ +package specs + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/sets" + + "yunion.io/x/onecloud/pkg/appctx" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +type specHandleFunc func(context context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (jsonutils.JSONObject, error) + +func AddSpecHandler(prefix string, app *appsrv.Application) { + // get models specs + for key, handleF := range map[string]specHandleFunc{ + "": models.GetAllModelSpecs, + "hosts": models.GetHostSpecs, + "isolated_devices": models.GetIsolatedDeviceSpecs, + "servers": models.GetServerSpecs, + } { + addModelSpecHandler(prefix, key, handleF, app) + } + + // get model objects by spec key + for key, handleF := range map[string]specQueryHandleFunc{ + "hosts": queryHosts, + "isolated_devices": queryIsolatedDevices, + } { + AddQuerySpecModelHandler(prefix, key, handleF, app) + } +} + +func processFilter(handleFunc specHandleFunc) appsrv.FilterHandler { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { + userCred := auth.FetchUserCredential(ctx) + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } + params := appctx.AppContextParams(ctx) + for key, v := range params { + query.(*jsonutils.JSONDict).Add(jsonutils.NewString(v), key) + } + spec, err := handleFunc(ctx, userCred, query.(*jsonutils.JSONDict)) + if err != nil { + httperrors.GeneralServerError(w, err) + return + } + ret := jsonutils.NewDict() + ret.Add(spec, "spec") + appsrv.SendJSON(w, ret) + } +} + +func addModelSpecHandler(prefix, managerPluralKey string, handleFunc specHandleFunc, app *appsrv.Application) { + af := auth.Authenticate(processFilter(handleFunc)) + name := "get_spec" + prefix = fmt.Sprintf("%s/specs", prefix) + if len(managerPluralKey) != 0 { + prefix = fmt.Sprintf("%s/%s", prefix, managerPluralKey) + name = fmt.Sprintf("get_%s_spec", managerPluralKey) + } + app.AddHandler2("GET", prefix, af, nil, name, nil) +} + +func AddQuerySpecModelHandler(prefix, managerPluralKey string, handleFunc specQueryHandleFunc, app *appsrv.Application) { + af := auth.Authenticate(processFilter(queryModelHandle(handleFunc))) + prefix = fmt.Sprintf("%s/specs/%s", prefix, managerPluralKey) + name := fmt.Sprintf("get_%s_spec_query", managerPluralKey) + app.AddHandler2("GET", fmt.Sprintf("%s//resource", prefix), af, nil, name, nil) +} + +func parseSpecKey(key string) ([]string, error) { + unEscapeStr, err := url.PathUnescape(key) + if err != nil { + return nil, err + } + return strings.Split(unEscapeStr, "/"), nil +} + +type specQueryHandleFunc func(context.Context, mcclient.TokenCredential, *jsonutils.JSONDict, []string) (jsonutils.JSONObject, error) + +func queryModelHandle(queryF specQueryHandleFunc) specHandleFunc { + return func(ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (jsonutils.JSONObject, error) { + specKey, _ := query.GetString("") + if specKey == "" { + return nil, httperrors.NewInputParameterError("Empty spec query key") + } + specKeys, err := parseSpecKey(specKey) + if err != nil { + return nil, httperrors.NewInputParameterError("Parse spec key %s error: %v", specKey, err) + } + return queryF(ctx, userCred, query, specKeys) + } +} + +func queryHosts( + ctx context.Context, + userCred mcclient.TokenCredential, + query *jsonutils.JSONDict, + specKeys []string, +) (jsonutils.JSONObject, error) { + gpuModels := []string{} + newSpecs := []string{} + gpuHostIds := []string{} + for _, specKey := range specKeys { + if specKey == "gpu_model" { + gpuModels = append(gpuModels, strings.Split(specKey, ":")[1]) + } else { + newSpecs = append(newSpecs, specKey) + } + } + + if len(gpuModels) != 0 { + devs, err := models.IsolatedDeviceManager.FindUnusedByModels(gpuModels) + if err != nil { + return nil, err + } + for _, dev := range devs { + gpuHostIds = append(gpuHostIds, dev.HostId) + } + } + + isOk := func(obj models.ISpecModel) bool { + if len(gpuHostIds) > 0 { + if !sets.NewString(gpuHostIds...).Has(obj.GetId()) { + return false + } + gpus, _ := models.IsolatedDeviceManager.FindUnusedGpusOnHost(obj.GetId()) + if len(gpus) == 0 { + return false + } + hostGpuModels := []string{} + for _, gpu := range gpus { + hostGpuModels = append(hostGpuModels, gpu.Model) + } + if !sets.NewString(hostGpuModels...).IsSuperset(sets.NewString(gpuModels...)) { + return false + } + } + return true + } + return handleQueryModel(models.HostManager, ctx, userCred, query, specKeys, isOk) +} + +func queryIsolatedDevices( + ctx context.Context, + userCred mcclient.TokenCredential, + query *jsonutils.JSONDict, + specKeys []string, +) (jsonutils.JSONObject, error) { + return handleQueryModel(models.IsolatedDeviceManager, ctx, userCred, query, specKeys, nil) +} + +func handleQueryModel( + manager models.ISpecModelManager, + ctx context.Context, + userCred mcclient.TokenCredential, + query *jsonutils.JSONDict, + specKeys []string, + isOkF func(models.ISpecModel) bool, +) (jsonutils.JSONObject, error) { + objects, err := models.ListItems(manager, ctx, userCred, query) + if err != nil { + return nil, httperrors.NewInternalServerError("Get object error: %v", err) + } + if len(objects) == 0 { + ret := jsonutils.NewArray() + return ret, nil + } + objs := QueryObjects(manager, objects, specKeys, isOkF) + return QueryObjectsToJson(objs, ctx, userCred, query) +} + +func QueryObjects(manager models.ISpecModelManager, objs []models.ISpecModel, specKeys []string, isOkF func(models.ISpecModel) bool) []models.ISpecModel { + selectedObjs := make([]models.ISpecModel, 0) + for _, obj := range objs { + specs := obj.GetSpec(true) + if specs == nil { + continue + } + if isOkF != nil && !isOkF(obj) { + continue + } + specIdents := manager.GetSpecIdent(specs) + if len(specIdents) == 0 { + continue + } + if !sets.NewString(specIdents...).IsSuperset(sets.NewString(specKeys...)) { + continue + } + selectedObjs = append(selectedObjs, obj) + } + return selectedObjs +} + +func QueryObjectsToJson(objs []models.ISpecModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + ret := jsonutils.NewArray() + for _, obj := range objs { + jsonData := jsonutils.Marshal(obj) + jsonDict, ok := jsonData.(*jsonutils.JSONDict) + if !ok { + return nil, fmt.Errorf("Invalid model data structure, not a dict") + } + extraDict := obj.GetCustomizeColumns(ctx, userCred, query) + if extraDict != nil { + jsonDict.Update(extraDict) + } + ret.Add(jsonDict) + } + return ret, nil +} diff --git a/pkg/compute/tasks/eip_associate_task.go b/pkg/compute/tasks/eip_associate_task.go index 28a21017b8..44a18c5998 100644 --- a/pkg/compute/tasks/eip_associate_task.go +++ b/pkg/compute/tasks/eip_associate_task.go @@ -32,6 +32,11 @@ func (self *EipAssociateTask) OnInit(ctx context.Context, obj db.IStandaloneMode instanceId, _ := self.Params.GetString("instance_id") server := models.GuestManager.FetchGuestById(instanceId) + + if server.Status != models.VM_ASSOCIATE_EIP { + server.SetStatus(self.UserCred, models.VM_ASSOCIATE_EIP, "associate eip") + } + if server == nil { msg := fmt.Sprintf("fail to find server for instanceId %s", instanceId) eip.SetStatus(self.UserCred, models.EIP_STATUS_ASSOCIATE_FAIL, msg) @@ -57,5 +62,7 @@ func (self *EipAssociateTask) OnInit(ctx context.Context, obj db.IStandaloneMode eip.SetStatus(self.UserCred, models.EIP_STATUS_READY, "associate") + server.StartSyncstatus(ctx, self.UserCred, "") + self.SetStageComplete(ctx, nil) } diff --git a/pkg/compute/tasks/eip_dissociate_task.go b/pkg/compute/tasks/eip_dissociate_task.go index a9e9aa6cef..fc7d8fd600 100644 --- a/pkg/compute/tasks/eip_dissociate_task.go +++ b/pkg/compute/tasks/eip_dissociate_task.go @@ -22,33 +22,43 @@ func init() { func (self *EipDissociateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { eip := obj.(*models.SElasticip) - extEip, err := eip.GetIEip() - if err != nil { - msg := fmt.Sprintf("fail to find iEIP for eip %s", err) - eip.SetStatus(self.UserCred, models.EIP_STATUS_DISSOCIATE_FAIL, msg) - self.SetStageFailed(ctx, msg) - return - } + server := eip.GetAssociateVM() + if server != nil { - if len(extEip.GetAssociationExternalId()) > 0 { - err = extEip.Dissociate() + if server.Status != models.VM_DISSOCIATE_EIP { + server.SetStatus(self.UserCred, models.VM_DISSOCIATE_EIP, "dissociate eip") + } + + extEip, err := eip.GetIEip() if err != nil { - msg := fmt.Sprintf("fail to remote dissociate eip %s", err) + msg := fmt.Sprintf("fail to find iEIP for eip %s", err) eip.SetStatus(self.UserCred, models.EIP_STATUS_DISSOCIATE_FAIL, msg) self.SetStageFailed(ctx, msg) return } - } - err = eip.Dissociate(ctx, self.UserCred) - if err != nil { - msg := fmt.Sprintf("fail to local dissociate eip %s", err) - eip.SetStatus(self.UserCred, models.EIP_STATUS_DISSOCIATE_FAIL, msg) - self.SetStageFailed(ctx, msg) - return - } + if len(extEip.GetAssociationExternalId()) > 0 { + err = extEip.Dissociate() + if err != nil { + msg := fmt.Sprintf("fail to remote dissociate eip %s", err) + eip.SetStatus(self.UserCred, models.EIP_STATUS_DISSOCIATE_FAIL, msg) + self.SetStageFailed(ctx, msg) + return + } + } - eip.SetStatus(self.UserCred, models.EIP_STATUS_READY, "dissociate") + err = eip.Dissociate(ctx, self.UserCred) + if err != nil { + msg := fmt.Sprintf("fail to local dissociate eip %s", err) + eip.SetStatus(self.UserCred, models.EIP_STATUS_DISSOCIATE_FAIL, msg) + self.SetStageFailed(ctx, msg) + return + } + + eip.SetStatus(self.UserCred, models.EIP_STATUS_READY, "dissociate") + + server.StartSyncstatus(ctx, self.UserCred, "") + } self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/guest_change_config_task.go b/pkg/compute/tasks/guest_change_config_task.go index 522d6305bd..b3938539ba 100644 --- a/pkg/compute/tasks/guest_change_config_task.go +++ b/pkg/compute/tasks/guest_change_config_task.go @@ -34,51 +34,51 @@ func (self *GuestChangeConfigTask) OnInit(ctx context.Context, obj db.IStandalon func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { iResizeDisks, err := self.Params.Get("resize") if iResizeDisks == nil || err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) return } resizeDisks := iResizeDisks.(*jsonutils.JSONArray) for i := 0; i < resizeDisks.Length(); i++ { iResizeSet, err := resizeDisks.GetAt(i) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } resizeSet := iResizeSet.(*jsonutils.JSONArray) diskId, err := resizeSet.GetAt(0) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } idStr, err := diskId.GetString() if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } jSize, err := resizeSet.GetAt(1) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } size, err := jSize.Int() if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } iDisk, err := models.DiskManager.FetchById(idStr) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(obj, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } disk := iDisk.(*models.SDisk) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(disk, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } @@ -86,7 +86,7 @@ func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, ob var pendingUsage models.SQuota err = self.GetPendingUsage(&pendingUsage) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(disk, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } @@ -120,7 +120,7 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob if iVcpuCount != nil { vcpuCount, err = iVcpuCount.Int() if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } @@ -128,15 +128,14 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob if iVmemSize != nil { vmemSize, err = iVmemSize.Int() if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } } err = guest.GetDriver().RequestChangeVmConfig(ctx, guest, self, vcpuCount, vmemSize) if err != nil { - //设置guest状态 - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } @@ -163,14 +162,14 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob return nil }) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } var pendingUsage models.SQuota err = self.GetPendingUsage(&pendingUsage) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } @@ -188,13 +187,13 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob err = models.QuotaManager.CancelPendingUsage(ctx, self.UserCred, guest.ProjectId, &pendingUsage, &cancelUsage) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } err = self.SetPendingUsage(&pendingUsage) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } @@ -202,7 +201,7 @@ func (self *GuestChangeConfigTask) OnCreateDisksComplete(ctx context.Context, ob self.SetStage("on_sync_status_complete", nil) err = guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) if err != nil { - self.SetStageFailed(ctx, err.Error()) + self.markStageFailed(obj, ctx, err.Error()) logclient.AddActionLog(guest, logclient.ACT_VM_CHANGE_FLAVOR, err, self.UserCred, false) return } @@ -228,3 +227,9 @@ func (self *GuestChangeConfigTask) OnGuestStartComplete(ctx context.Context, obj dt.Add(jsonutils.NewString(guest.Id), "id") self.SetStageComplete(ctx, dt) } + +func (self *GuestChangeConfigTask) markStageFailed(obj db.IStandaloneModel, ctx context.Context, reason string) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.UserCred, models.VM_CHANGE_FLAVOR_FAIL, reason) + self.SetStageFailed(ctx, reason) +} diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go index fbb84ea030..c053c9c1e0 100644 --- a/pkg/compute/tasks/guest_rebuild_root_task.go +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -139,9 +139,17 @@ func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Conte self.SetStage("OnGuestDeployComplete", nil) guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") - params := jsonutils.NewDict() - params.Set("reset_password", jsonutils.JSONTrue) - guest.StartGuestDeployTask(ctx, self.UserCred, params, "deploy", self.GetTaskId()) + // params := jsonutils.NewDict() + // params.Set("reset_password", jsonutils.JSONTrue) + + // clear logininfo + loginParams := make(map[string]interface{}) + loginParams["login_account"] = "none" + loginParams["login_key"] = "none" + loginParams["login_key_timestamp"] = "none" + guest.SetAllMetadata(ctx, loginParams, self.UserCred) + + guest.StartGuestDeployTask(ctx, self.UserCred, self.GetParams(), "deploy", self.GetTaskId()) } func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { diff --git a/pkg/httperrors/errors.go b/pkg/httperrors/errors.go index 4df2957513..662b59ec0e 100644 --- a/pkg/httperrors/errors.go +++ b/pkg/httperrors/errors.go @@ -100,6 +100,11 @@ func NewInputParameterError(msg string, params ...interface{}) *httputils.JSONCl return NewJsonClientError(400, "InputParameterError", msg, err) } +func NewWeakPasswordError() *httputils.JSONClientError { + msg, err := errorMessage("password must be 12 chars of at least one digit, letter, uppercase letter and punctuate", nil) + return NewJsonClientError(400, "WeakPasswordError", msg, err) +} + func NewInsufficientResourceError(msg string, params ...interface{}) *httputils.JSONClientError { msg, err := errorMessage(msg, params) return NewJsonClientError(400, "InsufficientResourceError", msg, err) diff --git a/pkg/mcclient/modules/mod_res_results.go b/pkg/mcclient/modules/mod_res_results.go new file mode 100644 index 0000000000..a043e79043 --- /dev/null +++ b/pkg/mcclient/modules/mod_res_results.go @@ -0,0 +1,13 @@ +package modules + +var ( + ResResults ResourceManager +) + +func init() { + ResResults = NewMeterManager("res_result", "res_results", + []string{"stat_month", "start_date", "end_date", "filter", "project_id"}, + []string{}, + ) + register(&ResResults) +} diff --git a/pkg/mcclient/options/servers.go b/pkg/mcclient/options/servers.go index 879424c738..c15a19e256 100644 --- a/pkg/mcclient/options/servers.go +++ b/pkg/mcclient/options/servers.go @@ -98,7 +98,7 @@ type ServerCreateOptions struct { Bios string `help:"BIOS" choices:"BIOS|UEFI"` Desc string `help:"Description" metavar:"" json:"description"` Boot string `help:"Boot device" metavar:"" choices:"disk|cdrom" json:"-"` - NoAccountInit *bool `help:"Not reset account password" json:"reset_password"` + NoAccountInit *bool `help:"Not reset account password"` AllowDelete *bool `help:"Unlock server to allow deleting" json:"-"` ShutdownBehavior string `help:"Behavior after VM server shutdown, stop or terminate server" metavar:"" choices:"stop|terminate"` AutoStart *bool `help:"Auto start server after it is created"` @@ -264,11 +264,12 @@ type ServerSaveImageOptions struct { } type ServerRebuildRootOptions struct { - ID string `help:"Server to rebuild root" json:"-"` - Image string `help:"New root Image template ID" json:"image_id"` - Keypair string `help:"ssh Keypair used for login"` - Password string `help:"Default user password"` - AutoStart *bool `help:"Auto start server after it is created"` + ID string `help:"Server to rebuild root" json:"-"` + Image string `help:"New root Image template ID" json:"image_id"` + Keypair string `help:"ssh Keypair used for login"` + Password string `help:"Default user password"` + NoAccountInit *bool `help:"Not reset account password"` + AutoStart *bool `help:"Auto start server after it is created"` } type ServerChangeConfigOptions struct { diff --git a/pkg/mcclient/session.go b/pkg/mcclient/session.go index 527e05c4e4..5cf2324f9e 100644 --- a/pkg/mcclient/session.go +++ b/pkg/mcclient/session.go @@ -21,6 +21,10 @@ const ( DEFAULT_API_VERSION = "v1" ) +var ( + MutilVersionService = []string{"compute"} +) + type ClientSession struct { client *Client region string @@ -72,7 +76,7 @@ func (this *ClientSession) GetServiceURL(service, endpointType string) (string, // session specific endpoint type should override the input endpointType, which is supplied by manager endpointType = this.endpointType } - if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { + if utils.IsInStringArray(service, MutilVersionService) && len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { service = fmt.Sprintf("%s_%s", service, this.apiVersion) } url, err := this.token.GetServiceURL(service, this.region, this.zone, endpointType) @@ -87,7 +91,7 @@ func (this *ClientSession) GetServiceURLs(service, endpointType string) ([]strin // session specific endpoint type should override the input endpointType, which is supplied by manager endpointType = this.endpointType } - if len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { + if utils.IsInStringArray(service, MutilVersionService) && len(this.apiVersion) > 0 && this.apiVersion != DEFAULT_API_VERSION { service = fmt.Sprintf("%s_%s", service, this.apiVersion) } urls, err := this.token.GetServiceURLs(service, this.region, this.zone, endpointType) diff --git a/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go index 9887628b77..2abf7694c4 100644 --- a/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go +++ b/pkg/scheduler/algorithm/predicates/baremetal/storage_predicate.go @@ -3,10 +3,10 @@ package baremetal import ( "fmt" - //"yunion.io/x/log" + "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" + "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" - "yunion.io/x/onecloud/pkg/scheduler/util/baremetal" ) type StoragePredicate struct { @@ -21,6 +21,27 @@ func (p *StoragePredicate) Clone() core.FitPredicate { return &StoragePredicate{} } +func toBaremetalDisks(disks []*api.Disk) []*baremetal.Disk { + ret := make([]*baremetal.Disk, len(disks)) + for i, disk := range disks { + ret[i] = &baremetal.Disk{ + Backend: disk.Backend, + ImageID: disk.ImageID, + Fs: disk.Fs, + Os: disk.Os, + OSDistribution: disk.OSDistribution, + Format: disk.Format, + MountPoint: disk.MountPoint, + Driver: disk.Driver, + Cache: disk.Cache, + ImageDiskFormat: disk.ImageDiskFormat, + Size: disk.Size, + Storage: disk.Storage, + } + } + return ret +} + func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := predicates.NewPredicateHelper(p, u, c) schedData := u.SchedData() @@ -35,7 +56,7 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor candidate.Storages, ) - if err == nil && baremetal.CheckDisksAllocable(layouts, schedData.Disks) { + if err == nil && baremetal.CheckDisksAllocable(layouts, toBaremetalDisks(schedData.Disks)) { h.SetCapacity(int64(1)) } else { h.SetCapacity(int64(0)) diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go index 85a7017066..96dc3560d6 100644 --- a/pkg/scheduler/api/sched.go +++ b/pkg/scheduler/api/sched.go @@ -9,8 +9,10 @@ import ( "github.com/bitly/go-simplejson" "yunion.io/x/log" - o "yunion.io/x/onecloud/cmd/scheduler/options" "yunion.io/x/pkg/utils" + + o "yunion.io/x/onecloud/cmd/scheduler/options" + "yunion.io/x/onecloud/pkg/compute/baremetal" ) type Meta map[string]string @@ -98,7 +100,7 @@ type SchedData struct { GroupRelations []GroupRelation `json:"group_relations"` // baremental - BaremetalDiskConfigs []*BaremetalDiskConfig `json:"baremetal_disk_config"` + BaremetalDiskConfigs []*baremetal.BaremetalDiskConfig `json:"baremetal_disk_config"` } func NewSchedData(sjson *simplejson.Json, count int64, byTest bool) (*SchedData, error) { @@ -543,8 +545,8 @@ func newDiskFromSimpleJson(sjson *simplejson.Json, byTest bool) (*Disk, error) { return disk, nil } -func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*BaremetalDiskConfig, error) { - baremetalDiskConfig := new(BaremetalDiskConfig) +func newBaremetalDiskConfigFromSimpleJson(sjson *simplejson.Json) (*baremetal.BaremetalDiskConfig, error) { + baremetalDiskConfig := new(baremetal.BaremetalDiskConfig) baremetalDiskConfig.Count = sjson.Get("count").MustInt64() baremetalDiskConfig.Conf = sjson.Get("conf").MustString() @@ -597,13 +599,13 @@ func (d *SchedData) fillDisksInfo(sjson *simplejson.Json, byTest bool) error { } func (d *SchedData) fillBaremetalDiskConfig(sjson *simplejson.Json) error { - baremetalDiskConfigs := []*BaremetalDiskConfig{} + baremetalDiskConfigs := []*baremetal.BaremetalDiskConfig{} if d.Hypervisor != HostTypeBaremetal { return nil } config, ok := sjson.CheckGet("baremetal_disk_config") if !ok { - defaultConfs := []*BaremetalDiskConfig{&BaremetalDefaultDiskConfig} + defaultConfs := []*baremetal.BaremetalDiskConfig{&baremetal.BaremetalDefaultDiskConfig} d.BaremetalDiskConfigs = defaultConfs log.V(4).Warningf("No baremetal_disk_config info found in json, use default baremetal disk config: %#v", defaultConfs) return nil diff --git a/pkg/scheduler/api/types.go b/pkg/scheduler/api/types.go index 12afec5291..8ad5257f97 100644 --- a/pkg/scheduler/api/types.go +++ b/pkg/scheduler/api/types.go @@ -25,31 +25,6 @@ const ( AggregateStrategyPrefer = "prefer" AggregateStrategyAvoid = "avoid" - // Baremetal related const - DISK_CONF_RAID0 = "raid0" - DISK_CONF_RAID1 = "raid1" - DISK_CONF_RAID5 = "raid5" - DISK_CONF_RAID10 = "raid10" - DISK_CONF_NONE = "none" - - DEFAULT_DISK_CONF = DISK_CONF_NONE - - DISK_TYPE_ROTATE = "rotate" - DISK_TYPE_SSD = "ssd" - DISK_TYPE_HYBRID = "hybrid" - - DEFAULT_DISK_TYPE = DISK_TYPE_ROTATE - - DISK_DRIVER_MEGARAID = "MegaRaid" - DISK_DRIVER_LINUX = "Linux" - DISK_DRIVER_HPSARAID = "HPSARaid" - DISK_DRIVER_MPT2SAS = "Mpt2SAS" - DISK_DRIVER_MARVELRAID = "MarvelRaid" - DISK_DRIVER_PCIE = "PCIE" - - HDD_DISK_SPEC_TYPE = "HDD" - SSD_DISK_SPEC_TYPE = "SSD" - // passthrough device type DIRECT_PCI_TYPE = "PCI" GPU_HPC_TYPE = "GPU-HPC" @@ -77,12 +52,6 @@ var ( HostTypeAzure, ) - BaremetalDefaultDiskConfig = BaremetalDiskConfig{ - Type: DISK_TYPE_HYBRID, - Conf: DISK_CONF_NONE, - Count: 0, - } - ValidGpuTypes = sets.NewString( GPU_HPC_TYPE, GPU_VGA_TYPE, @@ -114,34 +83,3 @@ func AggregateStrategyCheck(strategy string) (err error) { } return } - -type BaremetalStorage struct { - Slot int `json:"slot"` - Status string `json:"status"` - Rotate bool `json:"rotate"` - Adapter int `json:"adapter"` - Driver string `json:"driver"` - Model string `json:"model"` - Enclosure int `json:"enclousure"` - Size int64 `json:"size"` - MinStripSize int64 `json:"min_strip_size,omitempty"` - MaxStripSize int64 `json:"max_strip_size,omitempty"` - Index int64 `json:"index"` -} - -type BaremetalDiskConfig struct { - // disk type - Type string `json:"type"` - // raid config - Conf string `json:"conf"` - Count int64 `json:"count"` - Range []int64 `json:"range"` - Splits string `json:"splits"` - Adapter *int `json:"adapter"` - Driver string `json:"driver"` - Cachedbadbbu bool `json:"cachedbadbbu"` - Strip int64 `json:"strip"` - RA bool `json:"ra"` - WT bool `json:"wt"` - Direct bool `json:"direct"` -} diff --git a/pkg/scheduler/cache/candidate/baremetals.go b/pkg/scheduler/cache/candidate/baremetals.go index f7b584b242..56c7c99931 100644 --- a/pkg/scheduler/cache/candidate/baremetals.go +++ b/pkg/scheduler/cache/candidate/baremetals.go @@ -8,7 +8,7 @@ import ( fjson "github.com/json-iterator/go" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/scheduler/api" + "yunion.io/x/onecloud/pkg/compute/baremetal" "yunion.io/x/onecloud/pkg/scheduler/cache" "yunion.io/x/onecloud/pkg/scheduler/cache/db" "yunion.io/x/onecloud/pkg/scheduler/core" @@ -18,12 +18,12 @@ import ( type BaremetalDesc struct { baseHostDesc - Storages []*api.BaremetalStorage `json:"storages"` - StorageType string `json:"storage_type"` - StorageSize int64 `json:"storage_size"` - StorageInfo string `json:"storage_info"` - StorageDriver string `json:"storage_driver"` - ServerID string `json:"server_id"` + Storages []*baremetal.BaremetalStorage `json:"storages"` + StorageType string `json:"storage_type"` + StorageSize int64 `json:"storage_size"` + StorageInfo string `json:"storage_info"` + StorageDriver string `json:"storage_driver"` + ServerID string `json:"server_id"` } type BaremetalBuilder struct { @@ -234,7 +234,7 @@ func (bb *BaremetalBuilder) buildOne(bm *models.Host) (interface{}, error) { desc.HostType = bm.HostType desc.IsBaremetal = bm.IsBaremetal - var baremetalStorages []*api.BaremetalStorage + var baremetalStorages []*baremetal.BaremetalStorage err := fjson.Unmarshal([]byte(bm.StorageInfo), &baremetalStorages) if err != nil { // StorageInfo maybe is NULL diff --git a/pkg/scheduler/util/baremetal/doc.go b/pkg/scheduler/util/baremetal/doc.go deleted file mode 100644 index 90d29465b6..0000000000 --- a/pkg/scheduler/util/baremetal/doc.go +++ /dev/null @@ -1 +0,0 @@ -package baremetal // import "yunion.io/x/onecloud/pkg/scheduler/util/baremetal" diff --git a/pkg/util/aliyun/disk.go b/pkg/util/aliyun/disk.go index 111e36bfd2..cd688b4944 100644 --- a/pkg/util/aliyun/disk.go +++ b/pkg/util/aliyun/disk.go @@ -31,7 +31,7 @@ type SDisk struct { Description string DetachedTime time.Time Device string - DiskChargeType string + DiskChargeType InstanceChargeType DiskId string DiskName string EnableAutoSnapshot bool @@ -377,3 +377,19 @@ func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName } } + + +func (self *SDisk) GetBillingType() string { + switch self.DiskChargeType { + case PrePaidInstanceChargeType: + return models.BILLING_TYPE_PREPAID + case PostPaidInstanceChargeType: + return models.BILLING_TYPE_POSTPAID + default: + return models.BILLING_TYPE_PREPAID + } +} + +func (self *SDisk) GetExpiredAt() time.Time { + return self.ExpiredTime +} \ No newline at end of file diff --git a/pkg/util/aliyun/image.go b/pkg/util/aliyun/image.go index 343d9919d6..12e2948ec3 100644 --- a/pkg/util/aliyun/image.go +++ b/pkg/util/aliyun/image.go @@ -57,7 +57,20 @@ type SImage struct { } func (self *SImage) GetMetadata() *jsonutils.JSONDict { - return nil + data := jsonutils.NewDict() + if len(self.Architecture) > 0 { + data.Add(jsonutils.NewString(self.Architecture), "os_arch") + } + if len(self.OSType) > 0 { + data.Add(jsonutils.NewString(self.OSType), "os_name") + } + if len(self.Platform) > 0 { + data.Add(jsonutils.NewString(self.Platform), "os_distribution") + } + if len(self.OSName) > 0 { + data.Add(jsonutils.NewString(self.OSName), "os_version") + } + return data } func (self *SImage) GetId() string { diff --git a/pkg/util/aliyun/instance.go b/pkg/util/aliyun/instance.go index 8c675ff13f..9c18a922e1 100644 --- a/pkg/util/aliyun/instance.go +++ b/pkg/util/aliyun/instance.go @@ -156,6 +156,14 @@ func (self *SInstance) GetMetadata() *jsonutils.JSONDict { priceKey := fmt.Sprintf("%s::%s::%s::%s::%s", self.RegionId, self.InstanceType, self.InstanceNetworkType, self.OSType, optimized) data.Add(jsonutils.NewString(priceKey), "price_key") + if len(self.ImageId) > 0 { + if image, err := self.host.zone.region.GetImage(self.ImageId); err != nil { + log.Errorf("Failed to find image %s for instance %s", self.ImageId, self.GetName()) + } else if meta := image.GetMetadata(); meta != nil { + data.Update(meta) + } + } + return data } @@ -743,3 +751,18 @@ func (self *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { return nil, nil } } + +func (self *SInstance) GetBillingType() string { + switch self.InstanceChargeType { + case PrePaidInstanceChargeType: + return models.BILLING_TYPE_PREPAID + case PostPaidInstanceChargeType: + return models.BILLING_TYPE_POSTPAID + default: + return models.BILLING_TYPE_PREPAID + } +} + +func (self *SInstance) GetExpiredAt() time.Time { + return self.ExpiredTime +} diff --git a/pkg/util/aliyun/storagecache.go b/pkg/util/aliyun/storagecache.go index 60dcf0415b..96fa0ee606 100644 --- a/pkg/util/aliyun/storagecache.go +++ b/pkg/util/aliyun/storagecache.go @@ -88,10 +88,17 @@ func (self *SStoragecache) GetIImages() ([]cloudprovider.ICloudImage, error) { func (self *SStoragecache) UploadImage(userCred mcclient.TokenCredential, imageId string, osArch, osType, osDist string, extId string, isForce bool) (string, error) { if len(extId) > 0 { - status, _ := self.region.GetImageStatus(extId) + log.Debugf("UploadImage: Image external ID exists %s", extId) + + status, err := self.region.GetImageStatus(extId) + if err != nil { + log.Errorf("GetImageStatus error %s", err) + } if status == ImageStatusAvailable && !isForce { return extId, nil } + } else { + log.Debugf("UploadImage: no external ID") } return self.uploadImage(userCred, imageId, osArch, osType, osDist, isForce) diff --git a/pkg/util/azure/disk.go b/pkg/util/azure/disk.go index 085945ca00..25063052fe 100644 --- a/pkg/util/azure/disk.go +++ b/pkg/util/azure/disk.go @@ -270,3 +270,11 @@ func (self *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot func (self *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { return nil, cloudprovider.ErrNotImplemented } + +func (self *SDisk) GetBillingType() string { + return models.BILLING_TYPE_POSTPAID +} + +func (self *SDisk) GetExpiredAt() time.Time { + return time.Now() +} diff --git a/pkg/util/azure/instance.go b/pkg/util/azure/instance.go index 195d9fe83a..dcd2cb76a1 100644 --- a/pkg/util/azure/instance.go +++ b/pkg/util/azure/instance.go @@ -816,3 +816,11 @@ func (self *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { } return nil, nil } + +func (self *SInstance) GetBillingType() string { + return models.BILLING_TYPE_POSTPAID +} + +func (self *SInstance) GetExpiredAt() time.Time { + return time.Now() +} diff --git a/pkg/util/esxi/host.go b/pkg/util/esxi/host.go index 8b03a3814f..fa098fa474 100644 --- a/pkg/util/esxi/host.go +++ b/pkg/util/esxi/host.go @@ -361,4 +361,4 @@ func (self *SHost) CreateVM(name string, imgId string, sysDiskSize int, cpu int, passwd string, storageType string, diskSizes []int, publicKey string, secGrpId string) (cloudprovider.ICloudVM, error) { log.Debugf("CreateVM") return nil, cloudprovider.ErrNotImplemented -} +} \ No newline at end of file diff --git a/pkg/util/esxi/virtualmachine.go b/pkg/util/esxi/virtualmachine.go index 8996884109..680a28c654 100644 --- a/pkg/util/esxi/virtualmachine.go +++ b/pkg/util/esxi/virtualmachine.go @@ -237,3 +237,11 @@ func (self *SVirtualMachine) acquireVmrcUrl() (jsonutils.JSONObject, error) { func (dc *SVirtualMachine) ChangeConfig(instanceId string, ncpu int, vmem int) error { return cloudprovider.ErrNotImplemented } + +func (self *SVirtualMachine) GetBillingType() string { + return models.BILLING_TYPE_POSTPAID +} + +func (self *SVirtualMachine) GetExpiredAt() time.Time { + return time.Time{} +} \ No newline at end of file diff --git a/pkg/util/logclient/logclient.go b/pkg/util/logclient/logclient.go index e8a6ca1371..f7898ca1e7 100644 --- a/pkg/util/logclient/logclient.go +++ b/pkg/util/logclient/logclient.go @@ -3,12 +3,12 @@ package logclient import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/stringutils" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" - "yunion.io/x/pkg/util/stringutils" ) const ( @@ -102,7 +102,6 @@ func AddActionLog(model IObject, action string, iNotes interface{}, userCred mcc } logentry.Add(jsonutils.NewString(notes), "notes") - logclientWorkerMan.Run(func() { s := auth.GetSession(userCred, "", "") _, err := modules.Actions.Create(s, logentry) diff --git a/pkg/util/seclib2/seclib.go b/pkg/util/seclib2/seclib.go index d3fd831df3..ef1c0bb3cc 100644 --- a/pkg/util/seclib2/seclib.go +++ b/pkg/util/seclib2/seclib.go @@ -10,38 +10,82 @@ import ( const ( DIGITS = "23456789" LETTERS = "abcdefghjkmnpqrstuvwxyz" + UPPERS = "ABCDEFGHJKMNPRSTUVWXYZ" PUNC = "()~@#$%^&*-+={}[]:;<>,.?/" + + ALL_DIGITS = "0123456789" + ALL_LETTERS = "abcdefghijklmnopqrstuvwxyz" + ALL_UPPERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + ALL_PUNC = "~`!@#$%^&*()-_=+[]{}|:':\",./<>?" ) -var CHARS = fmt.Sprintf("%s%s%s%s", DIGITS, LETTERS, strings.ToUpper(LETTERS), PUNC) +type PasswordStrength struct { + Digits int + Lowercases int + Uppercases int + Punctuats int +} + +var CHARS = fmt.Sprintf("%s%s%s%s", DIGITS, LETTERS, UPPERS, PUNC) func RandomPassword2(width int) string { if width < 6 { width = 6 } for { + ps := PasswordStrength{} var buf bytes.Buffer - digitsCnt := 0 - letterCnt := 0 - upperCnt := 0 - puncCnt := 0 for i := 0; i < width; i += 1 { index := rand.Intn(len(CHARS)) ch := CHARS[index] if strings.IndexByte(DIGITS, ch) >= 0 { - digitsCnt += 1 + ps.Digits += 1 } else if strings.IndexByte(LETTERS, ch) >= 0 { - letterCnt += 1 - } else if strings.IndexByte(LETTERS, ch+32) >= 0 { - upperCnt += 1 + ps.Lowercases += 1 + } else if strings.IndexByte(UPPERS, ch) >= 0 { + ps.Uppercases += 1 } else if strings.IndexByte(PUNC, ch) >= 0 { - puncCnt += 1 + ps.Punctuats += 1 } buf.WriteByte(ch) } - if digitsCnt > 1 && letterCnt > 1 && upperCnt > 1 && puncCnt >= 1 && puncCnt <= 2 { + if ps.Digits > 1 && ps.Lowercases > 1 && ps.Uppercases > 1 && ps.Punctuats >= 1 && ps.Punctuats <= 2 { return buf.String() } } return "" } + +func AnalyzePasswordStrenth(passwd string) PasswordStrength { + ps := PasswordStrength{} + + for i := 0; i < len(passwd); i += 1 { + if strings.IndexByte(ALL_DIGITS, passwd[i]) >= 0 { + ps.Digits += 1 + } else if strings.IndexByte(ALL_LETTERS, passwd[i]) >= 0 { + ps.Lowercases += 1 + } else if strings.IndexByte(ALL_UPPERS, passwd[i]) >= 0 { + ps.Uppercases += 1 + } else if strings.IndexByte(ALL_PUNC, passwd[i]) >= 0 { + ps.Punctuats += 1 + } + } + return ps +} + +func (ps PasswordStrength) Len() int { + return ps.Punctuats + ps.Uppercases + ps.Lowercases + ps.Digits +} + +func (ps PasswordStrength) MeetComplexity() bool { + if ps.Punctuats > 0 && ps.Digits > 0 && ps.Lowercases > 0 && ps.Uppercases > 0 && ps.Len() >= 12 { + return true + } else { + return false + } +} + +func MeetComplxity(passwd string) bool { + ps := AnalyzePasswordStrenth(passwd) + return ps.MeetComplexity() +} diff --git a/pkg/util/seclib2/seclib_test.go b/pkg/util/seclib2/seclib_test.go new file mode 100644 index 0000000000..9921aa09fb --- /dev/null +++ b/pkg/util/seclib2/seclib_test.go @@ -0,0 +1,27 @@ +package seclib2 + +import ( + "testing" + "math/rand" + "time" +) + +func TestRandomPassword2(t *testing.T) { + rand.Seed(time.Now().Unix()) + t.Logf("%s", RandomPassword2(12)) +} + +func TestMeetComplxity(t *testing.T) { + cases := [] struct { + in string + want bool + } { + {"123456", false}, + {"123abcABC!@#", true}, + } + for _, c := range cases { + if c.want != MeetComplxity(c.in) { + t.Errorf("%s != %v", c.in, c.want) + } + } +}