From eaa03232d9c832aeccd155aaf2fe41bc763b18e0 Mon Sep 17 00:00:00 2001 From: rainzm Date: Wed, 26 Jan 2022 12:11:05 +0800 Subject: [PATCH 01/12] feat(region): add metadata prefix 'cls' Such tags have inherited and isolated properties --- pkg/apis/cloudcommon/db/metadata.go | 1 + pkg/apis/const.go | 2 + pkg/apis/input.go | 4 ++ pkg/cloudcommon/db/metadata.go | 18 +++--- pkg/cloudcommon/db/standalone_anon.go | 92 +++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 8 deletions(-) diff --git a/pkg/apis/cloudcommon/db/metadata.go b/pkg/apis/cloudcommon/db/metadata.go index 26294fe172..33ea11211c 100644 --- a/pkg/apis/cloudcommon/db/metadata.go +++ b/pkg/apis/cloudcommon/db/metadata.go @@ -20,4 +20,5 @@ const ( CLOUD_TAG_PREFIX = apis.CLOUD_TAG_PREFIX USER_TAG_PREFIX = apis.USER_TAG_PREFIX SYS_CLOUD_TAG_PREFIX = apis.SYS_CLOUD_TAG_PREFIX + CLASS_TAT_PREFIX = apis.CLASS_TAT_PREFIX ) diff --git a/pkg/apis/const.go b/pkg/apis/const.go index 55b8eb4a78..0beca5fb00 100644 --- a/pkg/apis/const.go +++ b/pkg/apis/const.go @@ -51,6 +51,8 @@ const ( CLOUD_TAG_PREFIX = "ext:" USER_TAG_PREFIX = "user:" SYS_CLOUD_TAG_PREFIX = "sys:" + // Such tags have inherited and isolated properties + CLASS_TAT_PREFIX = "cls:" ) var ( diff --git a/pkg/apis/input.go b/pkg/apis/input.go index a4155cabfc..323d273eb5 100644 --- a/pkg/apis/input.go +++ b/pkg/apis/input.go @@ -344,6 +344,10 @@ type PerformUserMetadataInput map[string]string // 全量替换资源的用户标签（元数据）输入 type PerformSetUserMetadataInput map[string]string +type PerformSetClassMetadataInput map[string]string + +type GetClassMetadataOutput map[string]string + // 获取资源的元数据输入 type GetMetadataInput struct { // 指定需要获取的所有标签的KEY列表，如果列表为空，则获取全部标签 diff --git a/pkg/cloudcommon/db/metadata.go b/pkg/cloudcommon/db/metadata.go index 630106a643..1882a3f713 100644 --- a/pkg/cloudcommon/db/metadata.go +++ b/pkg/cloudcommon/db/metadata.go @@ -45,6 +45,7 @@ const ( CLOUD_TAG_PREFIX = dbapi.CLOUD_TAG_PREFIX USER_TAG_PREFIX = dbapi.USER_TAG_PREFIX SYS_CLOUD_TAG_PREFIX = dbapi.SYS_CLOUD_TAG_PREFIX + CLASS_TAG_PREFIX = dbapi.CLASS_TAT_PREFIX // TAG_DELETE_RANGE_USER = "user" // TAG_DELETE_RANGE_CLOUD = CLOUD_TAG_PREFIX // "cloud" @@ -602,14 +603,15 @@ func (manager *SMetadataManager) rawSetValues(ctx context.Context, objType strin if replace { records := []SMetadata{} q := manager.Query().Equals("id", idStr).NotLike("key", `\_\_%`) //避免删除系统内置的metadata, _ 在mysql里面有特殊含义,需要转义 - switch replaceRange { - case USER_TAG_PREFIX: - q = q.Startswith("key", USER_TAG_PREFIX) - case CLOUD_TAG_PREFIX: - q = q.Startswith("key", CLOUD_TAG_PREFIX) - case SYS_CLOUD_TAG_PREFIX: - q = q.Startswith("key", SYS_CLOUD_TAG_PREFIX) - } + // switch replaceRange { + // case USER_TAG_PREFIX: + // q = q.Startswith("key", USER_TAG_PREFIX) + // case CLOUD_TAG_PREFIX: + // q = q.Startswith("key", CLOUD_TAG_PREFIX) + // case SYS_CLOUD_TAG_PREFIX: + // q = q.Startswith("key", SYS_CLOUD_TAG_PREFIX) + // } + q = q.Startswith("key", replaceRange) q = q.Filter(sqlchemy.NOT(sqlchemy.In(q.Field("key"), keys))) if err := FetchModelObjects(manager, q, &records); err != nil { log.Errorf("failed to fetch metadata error: %v", err) diff --git a/pkg/cloudcommon/db/standalone_anon.go b/pkg/cloudcommon/db/standalone_anon.go index 6911a9f209..68f2272991 100644 --- a/pkg/cloudcommon/db/standalone_anon.go +++ b/pkg/cloudcommon/db/standalone_anon.go @@ -29,6 +29,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/stringutils2" "yunion.io/x/onecloud/pkg/util/tagutils" @@ -63,6 +64,7 @@ func (model *SStandaloneAnonResourceBase) BeforeInsert() { type SStandaloneAnonResourceBaseManager struct { SResourceBaseManager SMetadataResourceBaseModelManager + SSecretResourceBaseModelManager } func NewStandaloneAnonResourceBaseManager( @@ -139,6 +141,8 @@ func (manager *SStandaloneAnonResourceBaseManager) ListItemFilter( q = manager.SMetadataResourceBaseModelManager.ListItemFilter(manager.GetIModelManager(), q, input.MetadataResourceListInput) + q = manager.SSecretResourceBaseModelManager.ListItemFilter(manager.GetIModelManager(), q, input.SecretResourceListInput) + return q, nil } @@ -274,6 +278,58 @@ func (model *SStandaloneAnonResourceBase) SetCloudMetadataAll(ctx context.Contex return Metadata.SetAll(ctx, model, userTags, userCred, USER_TAG_PREFIX) } +func (model *SStandaloneAnonResourceBase) SetClassMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { + afterCheck := make(map[string]interface{}, len(dictstore)) + for k, v := range dictstore { + if !strings.HasPrefix(k, CLASS_TAG_PREFIX) { + afterCheck[CLASS_TAG_PREFIX+k] = v + } else { + afterCheck[k] = v + } + } + err := Metadata.SetAll(ctx, model, afterCheck, userCred, CLASS_TAG_PREFIX) + if err != nil { + return errors.Wrap(err, "SetAll") + } + return nil +} + +func (model *SStandaloneAnonResourceBase) Inherit(ctx context.Context, sonModel *SStandaloneAnonResourceBase) error { + metadata, err := model.GetAllClassMetadata() + if err != nil { + return errors.Wrap(err, "GetAllPureMetadata") + } + if len(metadata) == 0 { + return nil + } + userCred := auth.AdminCredential() + dictstore := make(map[string]interface{}, 0) + for k, v := range metadata { + dictstore[k] = v + } + return sonModel.SetClassMetadataAll(ctx, dictstore, userCred) +} + +func (model *SStandaloneAnonResourceBase) IsInSameClass(ctx context.Context, pModel *SStandaloneAnonResourceBase) (bool, error) { + pureTags, err := model.GetAllClassMetadata() + if err != nil { + return false, errors.Wrap(err, "GetAllPureMetadata") + } + pureTagsP, err := pModel.GetAllClassMetadata() + if err != nil { + return false, errors.Wrap(err, "GetAllPureMetadata") + } + if len(pureTags) != len(pureTagsP) { + return false, nil + } + for k, v := range pureTags { + if vp, ok := pureTagsP[k]; !ok || vp != v { + return false, nil + } + } + return true, nil +} + func (model *SStandaloneAnonResourceBase) SetSysCloudMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { err := Metadata.SetAll(ctx, model, dictstore, userCred, SYS_CLOUD_TAG_PREFIX) if err != nil { @@ -326,6 +382,18 @@ func (model *SStandaloneAnonResourceBase) GetAllCloudMetadata() (map[string]stri return ret, nil } +func (model *SStandaloneAnonResourceBase) GetAllClassMetadata() (map[string]string, error) { + meta, err := Metadata.GetAll(nil, model, nil, CLASS_TAG_PREFIX, nil) + if err != nil { + return nil, errors.Wrap(err, "Metadata.GetAll") + } + ret := make(map[string]string) + for k, v := range meta { + ret[k[len(CLASS_TAG_PREFIX):]] = v + } + return ret, nil +} + // 获取资源标签（元数据） func (model *SStandaloneAnonResourceBase) GetDetailsMetadata(ctx context.Context, userCred mcclient.TokenCredential, input apis.GetMetadataInput) (apis.GetMetadataOutput, error) { val, err := Metadata.GetAll(ctx, model, input.Field, input.Prefix, userCred) @@ -393,6 +461,28 @@ func (model *SStandaloneAnonResourceBase) PerformSetUserMetadata(ctx context.Con return nil, nil } +func (model *SStandaloneAnonResourceBase) PerformSetClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformSetClassMetadataInput) (jsonutils.JSONObject, error) { + dictStore := make(map[string]interface{}) + for k, v := range input { + if len(k) > 64-len(CLASS_TAG_PREFIX) { + return nil, httperrors.NewInputParameterError("input key too long > %d", 64-len(CLASS_TAG_PREFIX)) + } + if len(v) > 65535 { + return nil, httperrors.NewInputParameterError("input value too long > %d", 65535) + } + dictStore[k] = v + } + err := model.SetClassMetadataAll(ctx, dictStore, userCred) + if err != nil { + return nil, errors.Wrap(err, "SetUserMetadataAll") + } + return nil, nil +} + +func (model *SStandaloneAnonResourceBase) GetDetailsClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, input apis.GetClassMetadataOutput) (apis.GetClassMetadataOutput, error) { + return model.GetAllClassMetadata() +} + type sPolicyTags struct { PolicyObjectTags tagutils.TTagSetList `json:"policy_object_tags"` PolicyProjectTags tagutils.TTagSetList `json:"policy_project_tags"` @@ -487,10 +577,12 @@ func (manager *SStandaloneAnonResourceBaseManager) FetchCustomizeColumns( ret := make([]apis.StandaloneAnonResourceDetails, len(objs)) upperRet := manager.SResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) metaRet := manager.SMetadataResourceBaseModelManager.FetchCustomizeColumns(manager.GetIModelManager(), userCred, objs, fields) + secretRet := manager.SSecretResourceBaseModelManager.FetchCustomizeColumns(manager.GetIModelManager(), userCred, objs, fields) for i := range objs { ret[i] = apis.StandaloneAnonResourceDetails{ ResourceBaseDetails: upperRet[i], MetadataResourceInfo: metaRet[i], + SecretResourceInfo: secretRet[i], } } return ret From 27d70e9a30a77346bcb8c5bc62eeba3d50c9deeb Mon Sep 17 00:00:00 2001 From: rainzm Date: Wed, 26 Jan 2022 12:11:47 +0800 Subject: [PATCH 02/12] feat(region,image): add SecretResourceBase --- cmd/climc/shell/compute/hosts.go | 2 + cmd/climc/shell/image/images.go | 24 +++++++++++ pkg/apis/cloudcommon/db/secret.go | 19 +++++++++ pkg/apis/compute/api.go | 3 ++ pkg/apis/compute/host.go | 10 +++++ pkg/apis/image/image.go | 5 +++ pkg/apis/list.go | 6 +++ pkg/apis/output.go | 5 +++ pkg/cloudcommon/db/secretresource.go | 63 ++++++++++++++++++++++++++++ pkg/compute/models/hosts.go | 25 +++++++++++ pkg/image/models/images.go | 7 ++++ pkg/mcclient/options/compute/host.go | 19 +++++++++ 12 files changed, 188 insertions(+) create mode 100644 pkg/apis/cloudcommon/db/secret.go create mode 100644 pkg/cloudcommon/db/secretresource.go diff --git a/cmd/climc/shell/compute/hosts.go b/cmd/climc/shell/compute/hosts.go index 419ca00e57..f99827a06c 100644 --- a/cmd/climc/shell/compute/hosts.go +++ b/cmd/climc/shell/compute/hosts.go @@ -49,6 +49,8 @@ func init() { cmd.BatchDelete(&options.BaseIdsOptions{}) cmd.Perform("remove-all-netifs", &options.BaseIdOptions{}) cmd.Perform("probe-isolated-devices", &options.BaseIdOptions{}) + cmd.Perform("set-secret-level", &compute.HostSetSecretLevelOptions{}) + cmd.Perform("remove-secret-level", &compute.HostSetSecretLevelOptions{}) cmd.BatchPerform("enable", &options.BaseIdsOptions{}) cmd.BatchPerform("disable", &options.BaseIdsOptions{}) diff --git a/cmd/climc/shell/image/images.go b/cmd/climc/shell/image/images.go index 2941adf77c..9a34d31319 100644 --- a/cmd/climc/shell/image/images.go +++ b/cmd/climc/shell/image/images.go @@ -419,6 +419,30 @@ func init() { return nil }) + type ImageSetSecretLevelOptions struct { + ID string `json:"-"` + SecretLevel string `json:"secret_level"` + } + R(&ImageSetSecretLevelOptions{}, "image-set-secret-level", "Set secret level", func(s *mcclient.ClientSession, args *ImageSetSecretLevelOptions) error { + result, err := modules.Images.PerformAction(s, args.ID, "set-secret-level", jsonutils.Marshal(args)) + if err != nil { + return err + } + printObject(result) + return nil + }) + type ImageRemoveSecretLevelOptions struct { + ID string `json:"-"` + } + R(&ImageRemoveSecretLevelOptions{}, "image-remove-secret-level", "remove secret level", func(s *mcclient.ClientSession, args *ImageRemoveSecretLevelOptions) error { + result, err := modules.Images.PerformAction(s, args.ID, "remove-secret-level", nil) + if err != nil { + return err + } + printObject(result) + return nil + }) + type ImagePublicOptions struct { ID []string `help:"ID or name of image" json:"-"` Scope string `help:"sharing scope" choices:"system|domain|project"` diff --git a/pkg/apis/cloudcommon/db/secret.go b/pkg/apis/cloudcommon/db/secret.go new file mode 100644 index 0000000000..75c3036ebe --- /dev/null +++ b/pkg/apis/cloudcommon/db/secret.go @@ -0,0 +1,19 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +const ( + SECRET_KEY = "secret_level" +) diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 6d15e6ac24..f58f40a748 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -208,6 +208,9 @@ type DiskConfig struct { //swagger:ignore DiskId string `json:"disk_id"` + + //swagger: ignore + SecretLevel string `json:"secret_level"` } type IsolatedDeviceConfig struct { diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index c1b7049fd1..d70c14b489 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -392,6 +392,16 @@ type HostCreateInput struct { BootMode string `json:"boot_mode"` } +type HostSetSecretLevelInput struct { + // description: 秘密级别 + SecretLevel string `json:"secret_level"` + Force bool `json:"force"` +} + +type HostRemoveSecretLevelInput struct { + Force bool `json:"force"` +} + type HostUpdateInput struct { apis.EnabledStatusInfrasResourceBaseUpdateInput diff --git a/pkg/apis/image/image.go b/pkg/apis/image/image.go index 098d05af43..643ed29f70 100644 --- a/pkg/apis/image/image.go +++ b/pkg/apis/image/image.go @@ -106,5 +106,10 @@ type ImageUpdateStatusInput struct { Reason string `json:"reason"` } +type ImageSetSecretLevelInput struct { + // description: 秘密级别 + SecretLevel string `json:"secret_level"` +} + type PerformProbeInput struct { } diff --git a/pkg/apis/list.go b/pkg/apis/list.go index 880d35ee42..4369face7f 100644 --- a/pkg/apis/list.go +++ b/pkg/apis/list.go @@ -246,11 +246,17 @@ type MetadataResourceListInput struct { WithMeta *bool `json:"with_meta"` } +type SecretResourceListInput struct { + SecretLevel string +} + type StandaloneAnonResourceListInput struct { ResourceBaseListInput MetadataResourceListInput + SecretResourceListInput + // 显示所有的资源，包括模拟的资源 ShowEmulated *bool `json:"show_emulated" help:"show emulated resources" negative:"do not show emulated resources"` diff --git a/pkg/apis/output.go b/pkg/apis/output.go index e35c30ad5b..be3c17304f 100644 --- a/pkg/apis/output.go +++ b/pkg/apis/output.go @@ -133,6 +133,10 @@ type MetadataResourceInfo struct { Metadata map[string]string `json:"metadata"` } +type SecretResourceInfo struct { + SecretLevel string `json:"secret_level"` +} + type StatusDomainLevelUserResourceDetails struct { StatusDomainLevelResourceDetails @@ -151,6 +155,7 @@ type StandaloneAnonResourceDetails struct { ResourceBaseDetails MetadataResourceInfo + SecretResourceInfo } type StandaloneResourceDetails struct { diff --git a/pkg/cloudcommon/db/secretresource.go b/pkg/cloudcommon/db/secretresource.go new file mode 100644 index 0000000000..fef93977e3 --- /dev/null +++ b/pkg/cloudcommon/db/secretresource.go @@ -0,0 +1,63 @@ +package db + +import ( + "context" + + "yunion.io/x/log" + "yunion.io/x/sqlchemy" + + "yunion.io/x/onecloud/pkg/apis" + dbapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/db" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" + "yunion.io/x/onecloud/pkg/util/tagutils" +) + +type SSecretResourceBaseModelManager struct{} + +func (secret *SSecretResourceBaseModelManager) ListItemFilter(manager IModelManager, q *sqlchemy.SQuery, input apis.SecretResourceListInput) *sqlchemy.SQuery { + if input.SecretLevel == "" { + return q + } + inputTagFilters := tagutils.STagFilters{} + tag := tagutils.STag{ + Key: CLASS_TAG_PREFIX + dbapi.SECRET_KEY, + Value: input.SecretLevel, + } + var tagSet tagutils.TTagSet = []tagutils.STag{tag} + inputTagFilters.AddFilter(tagSet) + return ObjectIdQueryWithTagFilters(q, "id", manager.Keyword(), inputTagFilters) +} + +func (secret *SSecretResourceBaseModelManager) FetchCustomizeColumns(manager IModelManager, userCred mcclient.TokenCredential, objs []interface{}, fields stringutils2.SSortedStrings) []apis.SecretResourceInfo { + ret := make([]apis.SecretResourceInfo, len(objs)) + resIds := make([]string, len(objs)) + for i := range objs { + resIds[i] = GetModelIdstr(objs[i].(IModel)) + } + if fields == nil || fields.Contains("secret_level") { + q := Metadata.Query("id", "key", "value").Equals("key", CLASS_TAG_PREFIX+dbapi.SECRET_KEY) + metaKeyValues := make(map[string][]SMetadata) + err := FetchQueryObjectsByIds(q, "id", resIds, &metaKeyValues) + if err != nil { + log.Errorf("FetchQueryObjectsByIds metadata fail %s", err) + return ret + } + for i := range objs { + if metaList, ok := metaKeyValues[resIds[i]]; ok { + ret[i].SecretLevel = metaList[0].Value + } + } + } + return ret +} + +func (secret *SSecretResourceBaseModelManager) SetSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, model *SStandaloneAnonResourceBase, secretLevel string) error { + return model.SetClassMetadataAll(ctx, map[string]interface{}{ + dbapi.SECRET_KEY: secretLevel, + }, userCred) +} + +func (secret *SSecretResourceBaseModelManager) RemoveSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, model *SStandaloneAnonResourceBase) error { + return model.RemoveMetadata(ctx, CLASS_TAG_PREFIX+dbapi.SECRET_KEY, userCred) +} diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 4fe13c93bf..7d202b41ee 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -5801,6 +5801,31 @@ func (manager *SHostManager) initHostname() error { return nil } +func (host *SHost) PerformSetSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.HostSetSecretLevelInput) (jsonutils.JSONObject, error) { + if input.SecretLevel == "" { + return nil, httperrors.NewMissingParameterError("secret_level") + } + count, err := host.GetGuestCount() + if err != nil { + return nil, errors.Wrap(err, "unable to GetGuestCount") + } + if !input.Force && count > 0 { + return nil, httperrors.NewForbiddenError("It is forbidden to modify the security level of the host with virtual machines") + } + return nil, HostManager.SSecretResourceBaseModelManager.SetSecretLevel(ctx, userCred, &host.SStandaloneAnonResourceBase, input.SecretLevel) +} + +func (host *SHost) PerformRemoveSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.HostRemoveSecretLevelInput) (jsonutils.JSONObject, error) { + count, err := host.GetGuestCount() + if err != nil { + return nil, errors.Wrap(err, "unable to GetGuestCount") + } + if !input.Force && count > 0 { + return nil, httperrors.NewForbiddenError("It is forbidden to modify the security level of the host with virtual machines") + } + return nil, HostManager.SSecretResourceBaseModelManager.RemoveSecretLevel(ctx, userCred, &host.SStandaloneAnonResourceBase) +} + func (manager *SHostManager) InitializeData() error { return manager.initHostname() } diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 87f5433c54..2a2a1d9dab 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -1536,6 +1536,13 @@ func (img *SImage) PerformUpdateStatus(ctx context.Context, userCred mcclient.To return nil, nil } +func (img *SImage) PerformSetSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ImageSetSecretLevelInput) (jsonutils.JSONObject, error) { + if input.SecretLevel == "" { + return nil, httperrors.NewMissingParameterError("secret_level") + } + return nil, ImageManager.SSecretResourceBaseModelManager.SetSecretLevel(ctx, userCred, &img.SStandaloneAnonResourceBase, input.SecretLevel) +} + func (img *SImage) PerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) (jsonutils.JSONObject, error) { if img.IsGuestImage.IsTrue() { return nil, errors.Wrap(httperrors.ErrForbidden, "cannot perform public for guest image") diff --git a/pkg/mcclient/options/compute/host.go b/pkg/mcclient/options/compute/host.go index 12a8b67228..06bf6b923f 100644 --- a/pkg/mcclient/options/compute/host.go +++ b/pkg/mcclient/options/compute/host.go @@ -85,3 +85,22 @@ type HostStatusStatisticsOptions struct { HostListOptions options.StatusStatisticsOptions } + +type HostSetSecretLevelOptions struct { + options.BaseIdOptions + SecretLevel string `json:"secret_level"` + Force bool `json:"force"` +} + +func (opt *HostSetSecretLevelOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opt), nil +} + +type HostRemoveSecretLevelOptions struct { + options.BaseIdOptions + Force bool `json:"force"` +} + +func (opt *HostRemoveSecretLevelOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opt), nil +} From 9a2348e9185cc7cc3598c32827fccc375d3935cd Mon Sep 17 00:00:00 2001 From: rainzm Date: Wed, 26 Jan 2022 15:33:12 +0800 Subject: [PATCH 03/12] feat(region,image): cachedimage inherits the class metadata of image --- pkg/apis/compute/cachedimage.go | 8 ++ pkg/apis/image/consts.go | 2 + pkg/apis/image/image.go | 7 ++ pkg/compute/models/cachedimages.go | 36 +++++++++ pkg/image/models/images.go | 40 +++++++++- .../tasks/image_sync_classmetadata_task.go | 78 +++++++++++++++++++ pkg/util/logclient/consts.go | 2 + 7 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 pkg/image/tasks/image_sync_classmetadata_task.go diff --git a/pkg/apis/compute/cachedimage.go b/pkg/apis/compute/cachedimage.go index e10e7e65c1..7cbe4d2357 100644 --- a/pkg/apis/compute/cachedimage.go +++ b/pkg/apis/compute/cachedimage.go @@ -26,6 +26,10 @@ type CachedImageUncacheImageInput struct { IsForce bool `json:"is_force"` } +type CachedImageManagerCacheImageInput struct { + ImageId string `json:"image_id"` +} + type CachedimageDetails struct { apis.SharableVirtualResourceDetails @@ -54,3 +58,7 @@ type CachedimageDetails struct { // example: 0 CachedCount int `json:"cached_count"` } + +type CachedImageSetClassMetadataInput struct { + ClassMetadata map[string]string `json:"class_metadata"` +} diff --git a/pkg/apis/image/consts.go b/pkg/apis/image/consts.go index 6a8e0d947f..704d7602af 100644 --- a/pkg/apis/image/consts.go +++ b/pkg/apis/image/consts.go @@ -35,6 +35,8 @@ const ( IMAGE_STATUS_DELETED = "deleted" IMAGE_STATUS_PENDING_DELETE = "pending_delete" + IMAGE_STATUS_SYNC_CLASS_METADATA_FAILEd = "sync_class_metadata_failed" + ImageTypeTemplate = TImageType("image") ImageTypeISO = TImageType("iso") diff --git a/pkg/apis/image/image.go b/pkg/apis/image/image.go index 643ed29f70..116d47fee9 100644 --- a/pkg/apis/image/image.go +++ b/pkg/apis/image/image.go @@ -111,5 +111,12 @@ type ImageSetSecretLevelInput struct { SecretLevel string `json:"secret_level"` } +type ImageGetClassMetadataInput struct { +} + +type ImageGetClassMetadataOutput struct { + ClassMetadata map[string]string `json:"class_metadata"` +} + type PerformProbeInput struct { } diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index d3bc536fe5..d996206ef7 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -204,6 +204,25 @@ func (self *SCachedimage) GetImage() (*cloudprovider.SImage, error) { } } +func (self *SCachedimage) syncClassMetadata(ctx context.Context, userCred mcclient.TokenCredential) error { + session := auth.GetSessionWithInternal(ctx, userCred, "", "") + ret, err := image.Images.GetSpecific(session, self.Id, "class-metadata", nil) + if err != nil { + return errors.Wrap(err, "unable to get class_metadata") + } + classMetadata := make(map[string]string, 0) + err = ret.Unmarshal(&classMetadata) + if err != nil { + return err + } + + cm := make(map[string]interface{}, len(classMetadata)) + for k, v := range classMetadata { + cm[k] = v + } + return self.SetClassMetadataAll(ctx, cm, userCred) +} + func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SCachedimage, error) { lockman.LockRawObject(ctx, manager.Keyword(), "name") defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name") @@ -254,6 +273,7 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us } db.OpsLog.LogEvent(&imageCache, db.ACT_CREATE, info, userCred) + imageCache.syncClassMetadata(ctx, userCred) return &imageCache, nil } else { log.Errorf("fetching image cache (%s) failed: %s", img.Id, err) @@ -408,6 +428,22 @@ func (self *SCachedimage) PerformUncacheImage(ctx context.Context, userCred mccl return storagecache.PerformUncacheImage(ctx, userCred, query, jsonutils.Marshal(map[string]interface{}{"image": self.Id, "is_force": input.IsForce})) } +func (self *SCachedimageManager) PerformCacheImage(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.CachedImageManagerCacheImageInput) (jsonutils.JSONObject, error) { + if len(input.ImageId) == 0 { + return nil, httperrors.NewMissingParameterError("image_id") + } + s := auth.GetAdminSession(ctx, options.Options.Region, "") + obj, err := image.Images.Get(s, input.ImageId, nil) + if err != nil { + return nil, errors.Wrap(err, "modules.Images.Get") + } + _, err = self.cacheGlanceImageInfo(ctx, userCred, obj) + if err != nil { + return nil, errors.Wrap(err, "manager.cacheGlanceImageInfo") + } + return nil, nil +} + func (self *SCachedimage) addRefCount() { if self.GetStatus() != api.CACHED_IMAGE_STATUS_ACTIVE { return diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 2a2a1d9dab..508f99c5a4 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -1540,7 +1540,45 @@ func (img *SImage) PerformSetSecretLevel(ctx context.Context, userCred mcclient. if input.SecretLevel == "" { return nil, httperrors.NewMissingParameterError("secret_level") } - return nil, ImageManager.SSecretResourceBaseModelManager.SetSecretLevel(ctx, userCred, &img.SStandaloneAnonResourceBase, input.SecretLevel) + err := ImageManager.SSecretResourceBaseModelManager.SetSecretLevel(ctx, userCred, &img.SStandaloneAnonResourceBase, input.SecretLevel) + if err != nil { + return nil, err + } + task, err := taskman.TaskManager.NewTask(ctx, "ImageSyncClassMetadataTask", img, userCred, nil, "", "", nil) + if err != nil { + return nil, err + } else { + task.ScheduleRun(nil) + } + return nil, nil +} + +func (img *SImage) PerformRemoveSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input jsonutils.JSONObject) (jsonutils.JSONObject, error) { + err := ImageManager.SSecretResourceBaseModelManager.RemoveSecretLevel(ctx, userCred, &img.SStandaloneAnonResourceBase) + if err != nil { + return nil, err + } + task, err := taskman.TaskManager.NewTask(ctx, "ImageSyncClassMetadataTask", img, userCred, nil, "", "", nil) + if err != nil { + return nil, err + } else { + task.ScheduleRun(nil) + } + return nil, nil +} + +func (img *SImage) PerformSetClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformSetClassMetadataInput) (jsonutils.JSONObject, error) { + ret, err := img.SStandaloneAnonResourceBase.PerformSetClassMetadata(ctx, userCred, query, input) + if err != nil { + return ret, err + } + task, err := taskman.TaskManager.NewTask(ctx, "ImageSyncClassMetadataTask", img, userCred, nil, "", "", nil) + if err != nil { + return nil, err + } else { + task.ScheduleRun(nil) + } + return nil, nil } func (img *SImage) PerformPublic(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformPublicProjectInput) (jsonutils.JSONObject, error) { diff --git a/pkg/image/tasks/image_sync_classmetadata_task.go b/pkg/image/tasks/image_sync_classmetadata_task.go new file mode 100644 index 0000000000..54e9b09f60 --- /dev/null +++ b/pkg/image/tasks/image_sync_classmetadata_task.go @@ -0,0 +1,78 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/image" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/image/models" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type ImageSyncClassMetadataTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(ImageSyncClassMetadataTask{}) +} + +func (self *ImageSyncClassMetadataTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + img := obj.(*models.SImage) + cm, err := img.GetAllClassMetadata() + if err != nil { + self.taskFailed(ctx, img, jsonutils.NewString(err.Error())) + return + } + session := auth.GetAdminSession(ctx, "", "") + _, err = compute.Cachedimages.PerformAction(session, img.Id, "set-class-metadata", jsonutils.Marshal(cm)) + if err == nil { + self.taskSuccess(ctx, img) + return + } + if errors.Cause(err) != httperrors.ErrResourceNotFound { + self.taskFailed(ctx, img, jsonutils.NewString(err.Error())) + return + } + params := jsonutils.NewDict() + params.Set("image_id", jsonutils.NewString(img.Id)) + _, err = compute.Cachedimages.PerformClassAction(session, "cache-image", params) + if err != nil { + self.taskFailed(ctx, img, jsonutils.NewString(err.Error())) + return + } + self.taskSuccess(ctx, img) +} + +func (self *ImageSyncClassMetadataTask) taskFailed(ctx context.Context, image *models.SImage, reason jsonutils.JSONObject) { + reasonStr, _ := reason.GetString() + image.SetStatus(self.UserCred, api.IMAGE_STATUS_SYNC_CLASS_METADATA_FAILEd, reasonStr) + logclient.AddActionLogWithStartable(self, image, logclient.ACT_SYNC_CLASS_METADATA, reason, self.UserCred, false) + self.SetStageFailed(ctx, reason) +} + +func (self *ImageSyncClassMetadataTask) taskSuccess(ctx context.Context, image *models.SImage) { + logclient.AddActionLogWithStartable(self, image, logclient.ACT_SYNC_CLASS_METADATA, nil, self.UserCred, true) + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/util/logclient/consts.go b/pkg/util/logclient/consts.go index 833f34525d..e449c2f888 100644 --- a/pkg/util/logclient/consts.go +++ b/pkg/util/logclient/consts.go @@ -216,4 +216,6 @@ const ( ACT_RESTART_NETWORK = "restart_network" ACT_RECOVERY = "recovery" + + ACT_SYNC_CLASS_METADATA = "sync_class_metadata" ) From a97341a42e5de1a49d29e16b503a1e0cbdb71c64 Mon Sep 17 00:00:00 2001 From: rainzm Date: Wed, 26 Jan 2022 16:34:48 +0800 Subject: [PATCH 04/12] feat(scheduler): add class_metadata predicate --- pkg/apis/scheduler/api.go | 3 + .../predicates/class_metadata_predicate.go | 136 ++++++++++++++++++ pkg/scheduler/algorithmprovider/defaults.go | 1 + 3 files changed, 140 insertions(+) create mode 100644 pkg/scheduler/algorithm/predicates/class_metadata_predicate.go diff --git a/pkg/apis/scheduler/api.go b/pkg/apis/scheduler/api.go index 87a251e7ef..fc35475aa8 100644 --- a/pkg/apis/scheduler/api.go +++ b/pkg/apis/scheduler/api.go @@ -64,6 +64,9 @@ type ServerConfig struct { GroupRelations []*GroupRelation `json:"group_releations"` Groups interface{} `json:"groups"` Id string `json:"id"` + + InstanceSnapshotId string `json:"instance_snapshot_id"` + InstanceBackupId string `json:"instance_backup_id"` } // ScheduleInput used by scheduler sync-schedule/test/forecast api diff --git a/pkg/scheduler/algorithm/predicates/class_metadata_predicate.go b/pkg/scheduler/algorithm/predicates/class_metadata_predicate.go new file mode 100644 index 0000000000..62b47beee5 --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/class_metadata_predicate.go @@ -0,0 +1,136 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicates + +import ( + "context" + "database/sql" + "fmt" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/scheduler/core" +) + +type ClassMetadataPredicate struct { + BasePredicate + cacheImage *models.SCachedimage + //TODO GuestImage + snapshot *models.SSnapshot + instanceSnapshot *models.SInstanceSnapshot + backup *models.SDiskBackup + instanceBackup *models.SInstanceBackup +} + +func (p *ClassMetadataPredicate) Name() string { + return "class_metadata" +} + +func (p *ClassMetadataPredicate) Clone() core.FitPredicate { + return &ClassMetadataPredicate{ + cacheImage: p.cacheImage, + snapshot: p.snapshot, + instanceSnapshot: p.instanceSnapshot, + backup: p.backup, + instanceBackup: p.instanceBackup, + } +} + +func (p *ClassMetadataPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { + info := u.SchedData() + if len(info.InstanceSnapshotId) > 0 { + obj, err := models.InstanceSnapshotManager.FetchById(info.InstanceSnapshotId) + if err != nil { + return false, errors.Wrapf(err, "unable to fetch instanceSnapshot %s", info.InstanceSnapshotId) + } + p.instanceSnapshot = obj.(*models.SInstanceSnapshot) + return true, nil + } + if len(info.InstanceBackupId) > 0 { + obj, err := models.InstanceBackupManager.FetchById(info.InstanceBackupId) + if err != nil { + return false, errors.Wrapf(err, "unable to fetch instanceBackup %s", info.InstanceBackupId) + } + p.instanceBackup = obj.(*models.SInstanceBackup) + return true, nil + } + disks := info.Disks + if len(disks) == 0 { + return false, nil + } + switch { + case disks[0].ImageId != "": + obj, err := models.CachedimageManager.FetchById(disks[0].ImageId) + if err != nil { + // 忽略第一次上传到glance镜像后未缓存的记录 + if err == sql.ErrNoRows { + return false, nil + } + return false, errors.Wrapf(err, "unable to fetch cachedimage %s", disks[0].ImageId) + } + p.cacheImage = obj.(*models.SCachedimage) + return true, nil + case disks[0].SnapshotId != "": + obj, err := models.SnapshotManager.FetchById(disks[0].SnapshotId) + if err != nil { + return false, errors.Wrapf(err, "unable to fetch snapshot %s", disks[0].SnapshotId) + } + p.snapshot = obj.(*models.SSnapshot) + return true, nil + case disks[0].BackupId != "": + obj, err := models.DiskBackupManager.FetchById(disks[0].BackupId) + if err != nil { + return false, errors.Wrapf(err, "unable to fetch diskbackup %s", disks[0].BackupId) + } + p.backup = obj.(*models.SDiskBackup) + return true, nil + } + return false, nil +} + +func (p *ClassMetadataPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { + h := NewPredicateHelper(p, u, c) + host := c.Getter().Host() + ctx := context.Background() + resourceDesc := "" + var sara *db.SStandaloneAnonResourceBase + switch { + case p.cacheImage != nil: + sara = &p.cacheImage.SStandaloneAnonResourceBase + resourceDesc = fmt.Sprintf("image %s", p.cacheImage.GetName()) + case p.snapshot != nil: + sara = &p.snapshot.SStandaloneAnonResourceBase + resourceDesc = fmt.Sprintf("snapshot %s", p.snapshot.GetName()) + case p.backup != nil: + sara = &p.backup.SStandaloneAnonResourceBase + resourceDesc = fmt.Sprintf("backup %s", p.backup.GetName()) + case p.instanceBackup != nil: + sara = &p.instanceBackup.SStandaloneAnonResourceBase + resourceDesc = fmt.Sprintf("instance backup %s", p.instanceBackup.GetName()) + case p.instanceSnapshot != nil: + sara = &p.instanceSnapshot.SStandaloneAnonResourceBase + resourceDesc = fmt.Sprintf("instance snapshot %s", p.instanceSnapshot.GetName()) + } + ic, err := host.IsInSameClass(ctx, sara) + if err != nil { + return false, nil, errors.Wrap(err, "unable to determine whether they are in a class") + } + if !ic { + h.Exclude(fmt.Sprintf("The host doesn't have the same class metadata as the choosen %s.", resourceDesc)) + } + return h.GetResult() +} diff --git a/pkg/scheduler/algorithmprovider/defaults.go b/pkg/scheduler/algorithmprovider/defaults.go index 3e10cda8cd..dbd99945c9 100644 --- a/pkg/scheduler/algorithmprovider/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults.go @@ -35,6 +35,7 @@ func defaultPredicates() sets.String { factory.RegisterFitPredicate("d-GuestMigrateFilter", &predicateguest.MigratePredicate{}), factory.RegisterFitPredicate("e-GuestDomainFilter", &predicates.DomainPredicate{}), factory.RegisterFitPredicate("e-GuestImageFilter", &predicateguest.ImagePredicate{}), + factory.RegisterFitPredicate("f-ClassMetadataFilter", &predicates.ClassMetadataPredicate{}), //factory.RegisterFitPredicate("f-GuestGroupFilter", &predicateguest.GroupPredicate{}), factory.RegisterFitPredicate("g-GuestCPUFilter", &predicateguest.CPUPredicate{}), factory.RegisterFitPredicate("h-GuestMemoryFilter", &predicateguest.MemoryPredicate{}), From e0377387008b5f34dcc32e39681901996f4362ea Mon Sep 17 00:00:00 2001 From: rainzm Date: Fri, 28 Jan 2022 14:27:08 +0800 Subject: [PATCH 05/12] feat(region,image): inherit between resources host -> guest -> disk -> image/backup/snapshot | instance image/backup/snapshot -> image/backup/snapshot --- pkg/compute/models/backup.go | 8 ++++++++ pkg/compute/models/disks.go | 11 +++++++++++ pkg/compute/models/guest_actions.go | 20 ++++++++++++++++++++ pkg/compute/models/guests.go | 8 ++++++++ pkg/compute/models/snapshots.go | 8 ++++++++ pkg/compute/regiondrivers/kvm.go | 9 +++++++++ pkg/image/models/image_guest.go | 18 ++++++++++++++++++ 7 files changed, 82 insertions(+) diff --git a/pkg/compute/models/backup.go b/pkg/compute/models/backup.go index e2c3185f0b..7acfaa9d8f 100644 --- a/pkg/compute/models/backup.go +++ b/pkg/compute/models/backup.go @@ -262,6 +262,14 @@ func (db *SDiskBackup) CustomizeCreate(ctx context.Context, userCred mcclient.To func (db *SDiskBackup) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { db.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) + disk, err := db.GetDisk() + if err != nil { + log.Errorf("unable to GetDisk: %s", err.Error()) + } + err = disk.Inherit(ctx, &db.SStandaloneAnonResourceBase) + if err != nil { + log.Errorf("unable to inherit from disk %s to backup %s: %s", disk.GetId(), db.GetId(), err.Error()) + } db.StartBackupCreateTask(ctx, userCred, nil, "") } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index d49766d132..32fdc7acdf 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -1044,6 +1044,17 @@ func (self *SDisk) PrepareSaveImage(ctx context.Context, userCred mcclient.Token if err != nil { return "", err } + // check class metadata + cm, err := self.GetAllClassMetadata() + if err != nil { + return "", errors.Wrap(err, "unable to GetAllClassMetadata") + } + if len(cm) > 0 { + _, err = image.Images.PerformAction(us, imageId, "set-class-metadata", jsonutils.Marshal(cm)) + if err != nil { + return "", errors.Wrapf(err, "unable to SetClassMetadata for image %s", imageId) + } + } return imageId, nil } diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index a739cda0cd..32f785687b 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -260,6 +260,18 @@ func (self *SGuest) PerformSaveGuestImage(ctx context.Context, userCred mcclient if err != nil { return nil, err } + guestImageId, _ := ret.GetString("id") + // set class metadata + cm, err := self.GetAllClassMetadata() + if err != nil { + return nil, errors.Wrap(err, "unable to GetAllClassMetadata") + } + if len(cm) > 0 { + _, err = image.Images.PerformAction(s, guestImageId, "set-class-metadata", jsonutils.Marshal(cm)) + if err != nil { + return nil, errors.Wrapf(err, "unable to SetClassMetadata for guest image %s", guestImageId) + } + } guestImageInfo := struct { RootImage imageapi.SubImageInfo DataImages []imageapi.SubImageInfo @@ -4695,6 +4707,10 @@ func (self *SGuest) PerformInstanceSnapshot( ctx, userCred, pendingUsage, pendingUsage, false) return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err) } + err = self.Inherit(ctx, &instanceSnapshot.SStandaloneAnonResourceBase) + if err != nil { + return nil, errors.Wrapf(err, "unable to inherit from guest %s to instance snapshot %s", self.GetId(), instanceSnapshot.GetId()) + } err = self.InstaceCreateSnapshot(ctx, userCred, instanceSnapshot, pendingUsage) if err != nil { quotas.CancelPendingUsage( @@ -4724,6 +4740,10 @@ func (self *SGuest) PerformInstanceBackup(ctx context.Context, userCred mcclient if err != nil { return nil, httperrors.NewInternalServerError("create instance backup failed: %s", err) } + err = self.Inherit(ctx, &instanceBackup.SStandaloneAnonResourceBase) + if err != nil { + return nil, errors.Wrapf(err, "unable to inherit from guest %s to instance backup %s", self.GetId(), instanceBackup.GetId()) + } err = self.InstanceCreateBackup(ctx, userCred, instanceBackup) if err != nil { return nil, httperrors.NewInternalServerError("start create backup task failed: %s", err) diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index f3f015afb5..3e62402762 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3790,6 +3790,10 @@ func (self *SGuest) createDiskOnHost( if autoAttach { err = self.attach2Disk(ctx, disk, userCred, diskConfig.Driver, diskConfig.Cache, diskConfig.Mountpoint) } + err = self.Inherit(ctx, &disk.SStandaloneAnonResourceBase) + if err != nil { + return nil, errors.Wrapf(err, "unable to inherit from guest %s to disk %s", self.GetId(), disk.GetId()) + } return disk, err } @@ -5267,6 +5271,10 @@ func (self *SGuest) OnScheduleToHost(ctx context.Context, userCred mcclient.Toke db.OpsLog.LogEvent(self, db.ACT_SCHEDULE, notes, userCred) host, _ := self.GetHost() + err = host.Inherit(ctx, &self.SStandaloneAnonResourceBase) + if err != nil { + return errors.Wrapf(err, "unable to inherit from host %s to guest %s", host.GetId(), self.GetId()) + } return host.ClearSchedDescCache() } diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index 4587493c22..fd36869827 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -407,6 +407,14 @@ func (snapshot *SSnapshot) PostCreate(ctx context.Context, userCred mcclient.Tok if err != nil { log.Errorf("quotas.CancelPendingUsage fail %s", err) } + disk, err := snapshot.GetDisk() + if err != nil { + log.Errorf("unable to GetDisk: %s", err.Error()) + } + err = disk.Inherit(ctx, &snapshot.SStandaloneAnonResourceBase) + if err != nil { + log.Errorf("unable to inherit from disk %s to snapshot %s: %s", disk.GetId(), snapshot.GetId(), err.Error()) + } } func (manager *SSnapshotManager) OnCreateComplete(ctx context.Context, items []db.IModel, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { diff --git a/pkg/compute/regiondrivers/kvm.go b/pkg/compute/regiondrivers/kvm.go index 8845c673b3..2ab87257d1 100644 --- a/pkg/compute/regiondrivers/kvm.go +++ b/pkg/compute/regiondrivers/kvm.go @@ -1127,6 +1127,11 @@ func (self *SKVMRegionDriver) RequestCreateInstanceSnapshot(ctx context.Context, return err } + err = isp.Inherit(ctx, &snapshot.SStandaloneAnonResourceBase) + if err != nil { + return errors.Wrapf(err, "unable to inherit from instance snapshot %s to snapshot %s", isp.GetId(), snapshot.GetId()) + } + err = models.InstanceSnapshotJointManager.CreateJoint(ctx, isp.Id, snapshot.Id, int8(diskIndex)) if err != nil { return err @@ -1274,6 +1279,10 @@ func (self *SKVMRegionDriver) RequestCreateInstanceBackup(ctx context.Context, g if err != nil { return err } + err = ib.Inherit(ctx, &backup.SStandaloneAnonResourceBase) + if err != nil { + return errors.Wrapf(err, "unable to inherit from instance backup %s to backup %s", ib.GetId(), backup.GetId()) + } err = models.InstanceBackupJointManager.CreateJoint(ctx, ib.Id, backup.Id, int8(i)) if err != nil { return err diff --git a/pkg/image/models/image_guest.go b/pkg/image/models/image_guest.go index f6c6651da0..88362e4db6 100644 --- a/pkg/image/models/image_guest.go +++ b/pkg/image/models/image_guest.go @@ -652,3 +652,21 @@ func (gi *SGuestImage) GetUsages() []db.IUsage { } return usages } + +func (img *SGuestImage) PerformSetClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformSetClassMetadataInput) (jsonutils.JSONObject, error) { + ret, err := img.SStandaloneAnonResourceBase.PerformSetClassMetadata(ctx, userCred, query, input) + if err != nil { + return ret, err + } + images, err := GuestImageJointManager.GetImagesByGuestImageId(img.Id) + if err != nil { + return nil, err + } + for i := range images { + _, err := images[i].PerformSetClassMetadata(ctx, userCred, query, input) + if err != nil { + return nil, errors.Wrapf(err, "unable to PerformSetClassMetadata for image %s", images[i].GetId()) + } + } + return nil, nil +} From a75d5f3099f7e42fcb0e9e3267faac050d9575db Mon Sep 17 00:00:00 2001 From: rainzm Date: Fri, 28 Jan 2022 18:58:17 +0800 Subject: [PATCH 06/12] feat(region): check class_metadate when attach disk to guest --- pkg/compute/models/guest_actions.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 32f785687b..e583343b38 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -731,6 +731,13 @@ func (self *SGuest) ValidateAttachDisk(ctx context.Context, disk *SDisk) error { if !utils.IsInStringArray(self.Status, guestStatus) { return httperrors.NewInputParameterError("Guest %s not support attach disk in status %s", self.Name, self.Status) } + ok, err := self.IsInSameClass(ctx, &disk.SStandaloneAnonResourceBase) + if err != nil { + return err + } + if !ok { + return httperrors.NewForbiddenError("the class metadata of guest and disk is different") + } return nil } From 4ba8d257baf279269a32a5eb3878b1289a56ccf5 Mon Sep 17 00:00:00 2001 From: rainzm Date: Mon, 7 Feb 2022 15:25:28 +0800 Subject: [PATCH 07/12] feat(region): add class-metadata for standaloneAnonResource --- pkg/apis/input.go | 2 ++ pkg/cloudcommon/db/standalone_anon.go | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/pkg/apis/input.go b/pkg/apis/input.go index 323d273eb5..47e3016e01 100644 --- a/pkg/apis/input.go +++ b/pkg/apis/input.go @@ -344,6 +344,8 @@ type PerformUserMetadataInput map[string]string // 全量替换资源的用户标签（元数据）输入 type PerformSetUserMetadataInput map[string]string +type PerformClassMetadataInput map[string]string + type PerformSetClassMetadataInput map[string]string type GetClassMetadataOutput map[string]string diff --git a/pkg/cloudcommon/db/standalone_anon.go b/pkg/cloudcommon/db/standalone_anon.go index 68f2272991..62ad3d5ee6 100644 --- a/pkg/cloudcommon/db/standalone_anon.go +++ b/pkg/cloudcommon/db/standalone_anon.go @@ -278,6 +278,14 @@ func (model *SStandaloneAnonResourceBase) SetCloudMetadataAll(ctx context.Contex return Metadata.SetAll(ctx, model, userTags, userCred, USER_TAG_PREFIX) } +func (model *SStandaloneAnonResourceBase) SetClassMetadataValues(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { + err := Metadata.SetValuesWithLog(ctx, model, dictstore, userCred) + if err != nil { + return errors.Wrap(err, "SetValuesWithLog") + } + return nil +} + func (model *SStandaloneAnonResourceBase) SetClassMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { afterCheck := make(map[string]interface{}, len(dictstore)) for k, v := range dictstore { @@ -461,6 +469,20 @@ func (model *SStandaloneAnonResourceBase) PerformSetUserMetadata(ctx context.Con return nil, nil } +// 更新资源的 class 标签 +func (model *SStandaloneAnonResourceBase) PerformClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformClassMetadataInput) (jsonutils.JSONObject, error) { + dictStore := make(map[string]interface{}) + for k, v := range input { + dictStore[CLASS_TAG_PREFIX+k] = v + } + err := model.SetUserMetadataValues(ctx, dictStore, userCred) + if err != nil { + return nil, errors.Wrap(err, "SetUserMetadataValues") + } + return nil, nil +} + +// 全量替换资源的所有 class 标签 func (model *SStandaloneAnonResourceBase) PerformSetClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformSetClassMetadataInput) (jsonutils.JSONObject, error) { dictStore := make(map[string]interface{}) for k, v := range input { From 73f8987d02d2093e9c985fb626cee0aa0ed3446d Mon Sep 17 00:00:00 2001 From: rainzm Date: Mon, 7 Feb 2022 16:00:13 +0800 Subject: [PATCH 08/12] fix(region): remove secret level resource --- cmd/climc/shell/compute/hosts.go | 2 - cmd/climc/shell/image/images.go | 24 ---------- pkg/apis/cloudcommon/db/secret.go | 19 -------- pkg/apis/compute/api.go | 3 -- pkg/apis/compute/host.go | 10 ----- pkg/apis/image/image.go | 12 ----- pkg/apis/list.go | 6 --- pkg/apis/output.go | 5 --- pkg/cloudcommon/db/secretresource.go | 63 --------------------------- pkg/cloudcommon/db/standalone_anon.go | 5 --- pkg/compute/models/hosts.go | 25 ----------- pkg/image/models/images.go | 31 ------------- pkg/mcclient/options/compute/host.go | 19 -------- 13 files changed, 224 deletions(-) delete mode 100644 pkg/apis/cloudcommon/db/secret.go delete mode 100644 pkg/cloudcommon/db/secretresource.go diff --git a/cmd/climc/shell/compute/hosts.go b/cmd/climc/shell/compute/hosts.go index f99827a06c..419ca00e57 100644 --- a/cmd/climc/shell/compute/hosts.go +++ b/cmd/climc/shell/compute/hosts.go @@ -49,8 +49,6 @@ func init() { cmd.BatchDelete(&options.BaseIdsOptions{}) cmd.Perform("remove-all-netifs", &options.BaseIdOptions{}) cmd.Perform("probe-isolated-devices", &options.BaseIdOptions{}) - cmd.Perform("set-secret-level", &compute.HostSetSecretLevelOptions{}) - cmd.Perform("remove-secret-level", &compute.HostSetSecretLevelOptions{}) cmd.BatchPerform("enable", &options.BaseIdsOptions{}) cmd.BatchPerform("disable", &options.BaseIdsOptions{}) diff --git a/cmd/climc/shell/image/images.go b/cmd/climc/shell/image/images.go index 9a34d31319..2941adf77c 100644 --- a/cmd/climc/shell/image/images.go +++ b/cmd/climc/shell/image/images.go @@ -419,30 +419,6 @@ func init() { return nil }) - type ImageSetSecretLevelOptions struct { - ID string `json:"-"` - SecretLevel string `json:"secret_level"` - } - R(&ImageSetSecretLevelOptions{}, "image-set-secret-level", "Set secret level", func(s *mcclient.ClientSession, args *ImageSetSecretLevelOptions) error { - result, err := modules.Images.PerformAction(s, args.ID, "set-secret-level", jsonutils.Marshal(args)) - if err != nil { - return err - } - printObject(result) - return nil - }) - type ImageRemoveSecretLevelOptions struct { - ID string `json:"-"` - } - R(&ImageRemoveSecretLevelOptions{}, "image-remove-secret-level", "remove secret level", func(s *mcclient.ClientSession, args *ImageRemoveSecretLevelOptions) error { - result, err := modules.Images.PerformAction(s, args.ID, "remove-secret-level", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - type ImagePublicOptions struct { ID []string `help:"ID or name of image" json:"-"` Scope string `help:"sharing scope" choices:"system|domain|project"` diff --git a/pkg/apis/cloudcommon/db/secret.go b/pkg/apis/cloudcommon/db/secret.go deleted file mode 100644 index 75c3036ebe..0000000000 --- a/pkg/apis/cloudcommon/db/secret.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package db - -const ( - SECRET_KEY = "secret_level" -) diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index f58f40a748..6d15e6ac24 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -208,9 +208,6 @@ type DiskConfig struct { //swagger:ignore DiskId string `json:"disk_id"` - - //swagger: ignore - SecretLevel string `json:"secret_level"` } type IsolatedDeviceConfig struct { diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index d70c14b489..c1b7049fd1 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -392,16 +392,6 @@ type HostCreateInput struct { BootMode string `json:"boot_mode"` } -type HostSetSecretLevelInput struct { - // description: 秘密级别 - SecretLevel string `json:"secret_level"` - Force bool `json:"force"` -} - -type HostRemoveSecretLevelInput struct { - Force bool `json:"force"` -} - type HostUpdateInput struct { apis.EnabledStatusInfrasResourceBaseUpdateInput diff --git a/pkg/apis/image/image.go b/pkg/apis/image/image.go index 116d47fee9..098d05af43 100644 --- a/pkg/apis/image/image.go +++ b/pkg/apis/image/image.go @@ -106,17 +106,5 @@ type ImageUpdateStatusInput struct { Reason string `json:"reason"` } -type ImageSetSecretLevelInput struct { - // description: 秘密级别 - SecretLevel string `json:"secret_level"` -} - -type ImageGetClassMetadataInput struct { -} - -type ImageGetClassMetadataOutput struct { - ClassMetadata map[string]string `json:"class_metadata"` -} - type PerformProbeInput struct { } diff --git a/pkg/apis/list.go b/pkg/apis/list.go index 4369face7f..880d35ee42 100644 --- a/pkg/apis/list.go +++ b/pkg/apis/list.go @@ -246,17 +246,11 @@ type MetadataResourceListInput struct { WithMeta *bool `json:"with_meta"` } -type SecretResourceListInput struct { - SecretLevel string -} - type StandaloneAnonResourceListInput struct { ResourceBaseListInput MetadataResourceListInput - SecretResourceListInput - // 显示所有的资源，包括模拟的资源 ShowEmulated *bool `json:"show_emulated" help:"show emulated resources" negative:"do not show emulated resources"` diff --git a/pkg/apis/output.go b/pkg/apis/output.go index be3c17304f..e35c30ad5b 100644 --- a/pkg/apis/output.go +++ b/pkg/apis/output.go @@ -133,10 +133,6 @@ type MetadataResourceInfo struct { Metadata map[string]string `json:"metadata"` } -type SecretResourceInfo struct { - SecretLevel string `json:"secret_level"` -} - type StatusDomainLevelUserResourceDetails struct { StatusDomainLevelResourceDetails @@ -155,7 +151,6 @@ type StandaloneAnonResourceDetails struct { ResourceBaseDetails MetadataResourceInfo - SecretResourceInfo } type StandaloneResourceDetails struct { diff --git a/pkg/cloudcommon/db/secretresource.go b/pkg/cloudcommon/db/secretresource.go deleted file mode 100644 index fef93977e3..0000000000 --- a/pkg/cloudcommon/db/secretresource.go +++ /dev/null @@ -1,63 +0,0 @@ -package db - -import ( - "context" - - "yunion.io/x/log" - "yunion.io/x/sqlchemy" - - "yunion.io/x/onecloud/pkg/apis" - dbapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/db" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/stringutils2" - "yunion.io/x/onecloud/pkg/util/tagutils" -) - -type SSecretResourceBaseModelManager struct{} - -func (secret *SSecretResourceBaseModelManager) ListItemFilter(manager IModelManager, q *sqlchemy.SQuery, input apis.SecretResourceListInput) *sqlchemy.SQuery { - if input.SecretLevel == "" { - return q - } - inputTagFilters := tagutils.STagFilters{} - tag := tagutils.STag{ - Key: CLASS_TAG_PREFIX + dbapi.SECRET_KEY, - Value: input.SecretLevel, - } - var tagSet tagutils.TTagSet = []tagutils.STag{tag} - inputTagFilters.AddFilter(tagSet) - return ObjectIdQueryWithTagFilters(q, "id", manager.Keyword(), inputTagFilters) -} - -func (secret *SSecretResourceBaseModelManager) FetchCustomizeColumns(manager IModelManager, userCred mcclient.TokenCredential, objs []interface{}, fields stringutils2.SSortedStrings) []apis.SecretResourceInfo { - ret := make([]apis.SecretResourceInfo, len(objs)) - resIds := make([]string, len(objs)) - for i := range objs { - resIds[i] = GetModelIdstr(objs[i].(IModel)) - } - if fields == nil || fields.Contains("secret_level") { - q := Metadata.Query("id", "key", "value").Equals("key", CLASS_TAG_PREFIX+dbapi.SECRET_KEY) - metaKeyValues := make(map[string][]SMetadata) - err := FetchQueryObjectsByIds(q, "id", resIds, &metaKeyValues) - if err != nil { - log.Errorf("FetchQueryObjectsByIds metadata fail %s", err) - return ret - } - for i := range objs { - if metaList, ok := metaKeyValues[resIds[i]]; ok { - ret[i].SecretLevel = metaList[0].Value - } - } - } - return ret -} - -func (secret *SSecretResourceBaseModelManager) SetSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, model *SStandaloneAnonResourceBase, secretLevel string) error { - return model.SetClassMetadataAll(ctx, map[string]interface{}{ - dbapi.SECRET_KEY: secretLevel, - }, userCred) -} - -func (secret *SSecretResourceBaseModelManager) RemoveSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, model *SStandaloneAnonResourceBase) error { - return model.RemoveMetadata(ctx, CLASS_TAG_PREFIX+dbapi.SECRET_KEY, userCred) -} diff --git a/pkg/cloudcommon/db/standalone_anon.go b/pkg/cloudcommon/db/standalone_anon.go index 62ad3d5ee6..5dfad8ff05 100644 --- a/pkg/cloudcommon/db/standalone_anon.go +++ b/pkg/cloudcommon/db/standalone_anon.go @@ -64,7 +64,6 @@ func (model *SStandaloneAnonResourceBase) BeforeInsert() { type SStandaloneAnonResourceBaseManager struct { SResourceBaseManager SMetadataResourceBaseModelManager - SSecretResourceBaseModelManager } func NewStandaloneAnonResourceBaseManager( @@ -141,8 +140,6 @@ func (manager *SStandaloneAnonResourceBaseManager) ListItemFilter( q = manager.SMetadataResourceBaseModelManager.ListItemFilter(manager.GetIModelManager(), q, input.MetadataResourceListInput) - q = manager.SSecretResourceBaseModelManager.ListItemFilter(manager.GetIModelManager(), q, input.SecretResourceListInput) - return q, nil } @@ -599,12 +596,10 @@ func (manager *SStandaloneAnonResourceBaseManager) FetchCustomizeColumns( ret := make([]apis.StandaloneAnonResourceDetails, len(objs)) upperRet := manager.SResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) metaRet := manager.SMetadataResourceBaseModelManager.FetchCustomizeColumns(manager.GetIModelManager(), userCred, objs, fields) - secretRet := manager.SSecretResourceBaseModelManager.FetchCustomizeColumns(manager.GetIModelManager(), userCred, objs, fields) for i := range objs { ret[i] = apis.StandaloneAnonResourceDetails{ ResourceBaseDetails: upperRet[i], MetadataResourceInfo: metaRet[i], - SecretResourceInfo: secretRet[i], } } return ret diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 7d202b41ee..4fe13c93bf 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -5801,31 +5801,6 @@ func (manager *SHostManager) initHostname() error { return nil } -func (host *SHost) PerformSetSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.HostSetSecretLevelInput) (jsonutils.JSONObject, error) { - if input.SecretLevel == "" { - return nil, httperrors.NewMissingParameterError("secret_level") - } - count, err := host.GetGuestCount() - if err != nil { - return nil, errors.Wrap(err, "unable to GetGuestCount") - } - if !input.Force && count > 0 { - return nil, httperrors.NewForbiddenError("It is forbidden to modify the security level of the host with virtual machines") - } - return nil, HostManager.SSecretResourceBaseModelManager.SetSecretLevel(ctx, userCred, &host.SStandaloneAnonResourceBase, input.SecretLevel) -} - -func (host *SHost) PerformRemoveSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.HostRemoveSecretLevelInput) (jsonutils.JSONObject, error) { - count, err := host.GetGuestCount() - if err != nil { - return nil, errors.Wrap(err, "unable to GetGuestCount") - } - if !input.Force && count > 0 { - return nil, httperrors.NewForbiddenError("It is forbidden to modify the security level of the host with virtual machines") - } - return nil, HostManager.SSecretResourceBaseModelManager.RemoveSecretLevel(ctx, userCred, &host.SStandaloneAnonResourceBase) -} - func (manager *SHostManager) InitializeData() error { return manager.initHostname() } diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 508f99c5a4..44e2c5c84e 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -1536,37 +1536,6 @@ func (img *SImage) PerformUpdateStatus(ctx context.Context, userCred mcclient.To return nil, nil } -func (img *SImage) PerformSetSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ImageSetSecretLevelInput) (jsonutils.JSONObject, error) { - if input.SecretLevel == "" { - return nil, httperrors.NewMissingParameterError("secret_level") - } - err := ImageManager.SSecretResourceBaseModelManager.SetSecretLevel(ctx, userCred, &img.SStandaloneAnonResourceBase, input.SecretLevel) - if err != nil { - return nil, err - } - task, err := taskman.TaskManager.NewTask(ctx, "ImageSyncClassMetadataTask", img, userCred, nil, "", "", nil) - if err != nil { - return nil, err - } else { - task.ScheduleRun(nil) - } - return nil, nil -} - -func (img *SImage) PerformRemoveSecretLevel(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input jsonutils.JSONObject) (jsonutils.JSONObject, error) { - err := ImageManager.SSecretResourceBaseModelManager.RemoveSecretLevel(ctx, userCred, &img.SStandaloneAnonResourceBase) - if err != nil { - return nil, err - } - task, err := taskman.TaskManager.NewTask(ctx, "ImageSyncClassMetadataTask", img, userCred, nil, "", "", nil) - if err != nil { - return nil, err - } else { - task.ScheduleRun(nil) - } - return nil, nil -} - func (img *SImage) PerformSetClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformSetClassMetadataInput) (jsonutils.JSONObject, error) { ret, err := img.SStandaloneAnonResourceBase.PerformSetClassMetadata(ctx, userCred, query, input) if err != nil { diff --git a/pkg/mcclient/options/compute/host.go b/pkg/mcclient/options/compute/host.go index 06bf6b923f..12a8b67228 100644 --- a/pkg/mcclient/options/compute/host.go +++ b/pkg/mcclient/options/compute/host.go @@ -85,22 +85,3 @@ type HostStatusStatisticsOptions struct { HostListOptions options.StatusStatisticsOptions } - -type HostSetSecretLevelOptions struct { - options.BaseIdOptions - SecretLevel string `json:"secret_level"` - Force bool `json:"force"` -} - -func (opt *HostSetSecretLevelOptions) Params() (jsonutils.JSONObject, error) { - return jsonutils.Marshal(opt), nil -} - -type HostRemoveSecretLevelOptions struct { - options.BaseIdOptions - Force bool `json:"force"` -} - -func (opt *HostRemoveSecretLevelOptions) Params() (jsonutils.JSONObject, error) { - return jsonutils.Marshal(opt), nil -} From d502c574b497f589af11c0aca43bc893c3be8475 Mon Sep 17 00:00:00 2001 From: rainzm Date: Mon, 7 Feb 2022 16:20:53 +0800 Subject: [PATCH 09/12] feat(climc): add (set-)class-metadata --- cmd/climc/shell/compute/hosts.go | 2 ++ cmd/climc/shell/image/images.go | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/cmd/climc/shell/compute/hosts.go b/cmd/climc/shell/compute/hosts.go index 419ca00e57..d7ebb6f3af 100644 --- a/cmd/climc/shell/compute/hosts.go +++ b/cmd/climc/shell/compute/hosts.go @@ -49,6 +49,8 @@ func init() { cmd.BatchDelete(&options.BaseIdsOptions{}) cmd.Perform("remove-all-netifs", &options.BaseIdOptions{}) cmd.Perform("probe-isolated-devices", &options.BaseIdOptions{}) + cmd.Perform("class-metadata", &options.ResourceMetadataOptions{}) + cmd.Perform("set-class-metadata", &options.ResourceMetadataOptions{}) cmd.BatchPerform("enable", &options.BaseIdsOptions{}) cmd.BatchPerform("disable", &options.BaseIdsOptions{}) diff --git a/cmd/climc/shell/image/images.go b/cmd/climc/shell/image/images.go index 2941adf77c..776258131b 100644 --- a/cmd/climc/shell/image/images.go +++ b/cmd/climc/shell/image/images.go @@ -497,6 +497,32 @@ func init() { return nil }) + R(&options.ResourceMetadataOptions{}, "image-set-class-metadata", "set class metadata for image", func(s *mcclient.ClientSession, opts *options.ResourceMetadataOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + img, err := modules.Images.PerformAction(s, opts.ID, "set-class-metadata", params) + if err != nil { + return err + } + printObject(img) + return nil + }) + + R(&options.ResourceMetadataOptions{}, "image-class-metadata", "set class metadata for image", func(s *mcclient.ClientSession, opts *options.ResourceMetadataOptions) error { + params, err := opts.Params() + if err != nil { + return err + } + img, err := modules.Images.PerformAction(s, opts.ID, "class-metadata", params) + if err != nil { + return err + } + printObject(img) + return nil + }) + type ImageProbeOptions struct { ID string `help:"ID or name of image to probe"` } From 42a8eb837bb37802358a6255229e9f4b514274ce Mon Sep 17 00:00:00 2001 From: rainzm Date: Tue, 15 Feb 2022 18:22:01 +0800 Subject: [PATCH 10/12] feat: add project with class metadata --- cmd/climc/shell/identity/projects.go | 2 + cmd/climc/shell/image/images.go | 26 ---- pkg/cloudcommon/db/standalone_anon.go | 24 +-- pkg/cloudcommon/db/virtualresource.go | 40 +++++ pkg/compute/models/cachedimages.go | 28 +++- pkg/compute/models/guests.go | 4 - .../predicates/class_metadata_predicate.go | 141 ++++++++++-------- pkg/scheduler/cache/candidate/base.go | 19 +++ pkg/scheduler/core/types.go | 2 + 9 files changed, 175 insertions(+), 111 deletions(-) diff --git a/cmd/climc/shell/identity/projects.go b/cmd/climc/shell/identity/projects.go index 2134af0a5f..a1ee148bd9 100644 --- a/cmd/climc/shell/identity/projects.go +++ b/cmd/climc/shell/identity/projects.go @@ -31,6 +31,8 @@ func init() { cmd.List(&identity_options.ProjectListOptions{}) cmd.Perform("user-metadata", &options.ResourceMetadataOptions{}) cmd.Perform("set-user-metadata", &options.ResourceMetadataOptions{}) + cmd.Perform("class-metadata", &options.ResourceMetadataOptions{}) + cmd.Perform("set-class-metadata", &options.ResourceMetadataOptions{}) cmd.GetProperty(&identity_options.ProjectGetPropertyTagValuePairOptions{}) cmd.GetProperty(&identity_options.ProjectGetPropertyTagValueTreeOptions{}) cmd.GetProperty(&identity_options.ProjectGetPropertyDomainTagValuePairOptions{}) diff --git a/cmd/climc/shell/image/images.go b/cmd/climc/shell/image/images.go index 776258131b..2941adf77c 100644 --- a/cmd/climc/shell/image/images.go +++ b/cmd/climc/shell/image/images.go @@ -497,32 +497,6 @@ func init() { return nil }) - R(&options.ResourceMetadataOptions{}, "image-set-class-metadata", "set class metadata for image", func(s *mcclient.ClientSession, opts *options.ResourceMetadataOptions) error { - params, err := opts.Params() - if err != nil { - return err - } - img, err := modules.Images.PerformAction(s, opts.ID, "set-class-metadata", params) - if err != nil { - return err - } - printObject(img) - return nil - }) - - R(&options.ResourceMetadataOptions{}, "image-class-metadata", "set class metadata for image", func(s *mcclient.ClientSession, opts *options.ResourceMetadataOptions) error { - params, err := opts.Params() - if err != nil { - return err - } - img, err := modules.Images.PerformAction(s, opts.ID, "class-metadata", params) - if err != nil { - return err - } - printObject(img) - return nil - }) - type ImageProbeOptions struct { ID string `help:"ID or name of image to probe"` } diff --git a/pkg/cloudcommon/db/standalone_anon.go b/pkg/cloudcommon/db/standalone_anon.go index 5dfad8ff05..d61f2659ca 100644 --- a/pkg/cloudcommon/db/standalone_anon.go +++ b/pkg/cloudcommon/db/standalone_anon.go @@ -283,7 +283,7 @@ func (model *SStandaloneAnonResourceBase) SetClassMetadataValues(ctx context.Con return nil } -func (model *SStandaloneAnonResourceBase) SetClassMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { +func (model *SStandaloneAnonResourceBase) SetClassMetadataAll(ctx context.Context, dictstore map[string]string, userCred mcclient.TokenCredential) error { afterCheck := make(map[string]interface{}, len(dictstore)) for k, v := range dictstore { if !strings.HasPrefix(k, CLASS_TAG_PREFIX) { @@ -308,19 +308,19 @@ func (model *SStandaloneAnonResourceBase) Inherit(ctx context.Context, sonModel return nil } userCred := auth.AdminCredential() - dictstore := make(map[string]interface{}, 0) - for k, v := range metadata { - dictstore[k] = v - } - return sonModel.SetClassMetadataAll(ctx, dictstore, userCred) + return sonModel.SetClassMetadataAll(ctx, metadata, userCred) } -func (model *SStandaloneAnonResourceBase) IsInSameClass(ctx context.Context, pModel *SStandaloneAnonResourceBase) (bool, error) { - pureTags, err := model.GetAllClassMetadata() +type IClassMetadataOwner interface { + GetAllClassMetadata() (map[string]string, error) +} + +func IsInSameClass(ctx context.Context, cmo1, cmo2 IClassMetadataOwner) (bool, error) { + pureTags, err := cmo1.GetAllClassMetadata() if err != nil { return false, errors.Wrap(err, "GetAllPureMetadata") } - pureTagsP, err := pModel.GetAllClassMetadata() + pureTagsP, err := cmo2.GetAllClassMetadata() if err != nil { return false, errors.Wrap(err, "GetAllPureMetadata") } @@ -335,6 +335,10 @@ func (model *SStandaloneAnonResourceBase) IsInSameClass(ctx context.Context, pMo return true, nil } +func (model *SStandaloneAnonResourceBase) IsInSameClass(ctx context.Context, pModel *SStandaloneAnonResourceBase) (bool, error) { + return IsInSameClass(ctx, model, pModel) +} + func (model *SStandaloneAnonResourceBase) SetSysCloudMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { err := Metadata.SetAll(ctx, model, dictstore, userCred, SYS_CLOUD_TAG_PREFIX) if err != nil { @@ -481,7 +485,7 @@ func (model *SStandaloneAnonResourceBase) PerformClassMetadata(ctx context.Conte // 全量替换资源的所有 class 标签 func (model *SStandaloneAnonResourceBase) PerformSetClassMetadata(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformSetClassMetadataInput) (jsonutils.JSONObject, error) { - dictStore := make(map[string]interface{}) + dictStore := make(map[string]string) for k, v := range input { if len(k) > 64-len(CLASS_TAG_PREFIX) { return nil, httperrors.NewInputParameterError("input key too long > %d", 64-len(CLASS_TAG_PREFIX)) diff --git a/pkg/cloudcommon/db/virtualresource.go b/pkg/cloudcommon/db/virtualresource.go index e2ae5cd40b..a366f67531 100644 --- a/pkg/cloudcommon/db/virtualresource.go +++ b/pkg/cloudcommon/db/virtualresource.go @@ -282,6 +282,18 @@ func (model *SVirtualResourceBase) CustomizeCreate(ctx context.Context, userCred return model.SStatusStandaloneResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data) } +func (model *SVirtualResourceBase) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { + project, err := model.GetTenantCache(ctx) + if err != nil { + log.Errorf("unable to GetTenantCache: %s", err.Error()) + return + } + err = project.Inherit(ctx, &model.SStandaloneAnonResourceBase) + if err != nil { + log.Errorf("unable to inherit class metadata from poject %s: %s", project.GetId(), err.Error()) + } +} + func (manager *SVirtualResourceBaseManager) FetchCustomizeColumns( ctx context.Context, userCred mcclient.TokenCredential, @@ -416,6 +428,27 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC log.Warningf("tenant_id %s not found", model.ProjectId) formerObj := NewTenant(model.ProjectId, "unknown", model.DomainId, "unknown") former = &formerObj + } else { + // check fromer's class metadata + cm, err := former.GetAllClassMetadata() + if err != nil { + return nil, errors.Wrap(err, "unable to GetAllClassMetadata") + } + if len(cm) > 0 { + return nil, httperrors.NewForbiddenError("can't change owner for resource in project with class metadata") + } + } + + toer, err := TenantCacheManager.FetchTenantById(ctx, ownerId.GetProjectId()) + if err != nil { + return nil, errors.Wrapf(err, "unable to get project %s", ownerId.GetProjectId()) + } + toCm, err := toer.GetAllClassMetadata() + if err != nil { + return nil, errors.Wrap(err, "unable to GetAllClassMetadata") + } + if model.Keyword() != "image" && len(toCm) > 0 { + return nil, httperrors.NewForbiddenError("can't change resource's owner as that in project with class metadata") } // clean shared projects before update project id @@ -461,6 +494,13 @@ func (model *SVirtualResourceBase) PerformChangeOwner(ctx context.Context, userC NewDomainId: ownerId.GetProjectDomainId(), NewDomain: ownerId.GetProjectDomain(), } + + // set class metadata + + err = model.SetClassMetadataAll(ctx, toCm, userCred) + if err != nil { + return nil, errors.Wrap(err, "unable to SetClassMetadataAll") + } logclient.AddActionLogWithContext(ctx, model, logclient.ACT_CHANGE_OWNER, notes, userCred, true) return nil, nil } diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index d996206ef7..f95499e3d2 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -216,11 +216,7 @@ func (self *SCachedimage) syncClassMetadata(ctx context.Context, userCred mcclie return err } - cm := make(map[string]interface{}, len(classMetadata)) - for k, v := range classMetadata { - cm[k] = v - } - return self.SetClassMetadataAll(ctx, cm, userCred) + return self.SetClassMetadataAll(ctx, classMetadata, userCred) } func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, userCred mcclient.TokenCredential, info jsonutils.JSONObject) (*SCachedimage, error) { @@ -317,6 +313,28 @@ func (image *SCachedimage) GetStorages() ([]SStorage, error) { return storages, nil } +func (manager *SCachedimageManager) GetCachedimageById(ctx context.Context, imageId string) (*SCachedimage, error) { + img, err := manager.FetchById(imageId) + if err == nil { + return img.(*SCachedimage), nil + } + if errors.Cause(err) != sql.ErrNoRows { + return nil, err + } + s := auth.GetAdminSession(ctx, options.Options.Region, "") + obj, err := image.Images.Get(s, imageId, nil) + if err != nil { + log.Errorf("GetImageById %s error %s", imageId, err) + return nil, errors.Wrap(err, "modules.Images.Get") + } + userCred := s.GetToken() + cachedImage, err := manager.cacheGlanceImageInfo(ctx, userCred, obj) + if err != nil { + return nil, errors.Wrap(err, "manager.cacheGlanceImageInfo") + } + return cachedImage, nil +} + func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*cloudprovider.SImage, error) { imgObj, _ := manager.FetchById(imageId) if imgObj != nil { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 3e62402762..ecad29457b 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -5271,10 +5271,6 @@ func (self *SGuest) OnScheduleToHost(ctx context.Context, userCred mcclient.Toke db.OpsLog.LogEvent(self, db.ACT_SCHEDULE, notes, userCred) host, _ := self.GetHost() - err = host.Inherit(ctx, &self.SStandaloneAnonResourceBase) - if err != nil { - return errors.Wrapf(err, "unable to inherit from host %s to guest %s", host.GetId(), self.GetId()) - } return host.ClearSchedDescCache() } diff --git a/pkg/scheduler/algorithm/predicates/class_metadata_predicate.go b/pkg/scheduler/algorithm/predicates/class_metadata_predicate.go index 62b47beee5..7eba908813 100644 --- a/pkg/scheduler/algorithm/predicates/class_metadata_predicate.go +++ b/pkg/scheduler/algorithm/predicates/class_metadata_predicate.go @@ -16,7 +16,6 @@ package predicates import ( "context" - "database/sql" "fmt" "yunion.io/x/pkg/errors" @@ -28,12 +27,23 @@ import ( type ClassMetadataPredicate struct { BasePredicate - cacheImage *models.SCachedimage - //TODO GuestImage - snapshot *models.SSnapshot - instanceSnapshot *models.SInstanceSnapshot - backup *models.SDiskBackup - instanceBackup *models.SInstanceBackup + + guestSource *ResourceWithClassMetadata + tenant *ResourceWithClassMetadata +} + +type ResourceWithClassMetadata struct { + keyword string + name string + classMedata map[string]string +} + +func (rcm *ResourceWithClassMetadata) GetAllClassMetadata() (map[string]string, error) { + return rcm.classMedata, nil +} + +func (rcm *ResourceWithClassMetadata) GetDescription() string { + return fmt.Sprintf("%s %s", rcm.keyword, rcm.name) } func (p *ClassMetadataPredicate) Name() string { @@ -42,95 +52,94 @@ func (p *ClassMetadataPredicate) Name() string { func (p *ClassMetadataPredicate) Clone() core.FitPredicate { return &ClassMetadataPredicate{ - cacheImage: p.cacheImage, - snapshot: p.snapshot, - instanceSnapshot: p.instanceSnapshot, - backup: p.backup, - instanceBackup: p.instanceBackup, + guestSource: p.guestSource, + tenant: p.tenant, } } func (p *ClassMetadataPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) { info := u.SchedData() - if len(info.InstanceSnapshotId) > 0 { + tenant, err := db.TenantCacheManager.FetchTenantById(context.Background(), info.Project) + if err != nil { + return false, errors.Wrapf(err, "unable to fetch tenant %s", info.Project) + } + tcm, err := tenant.GetAllClassMetadata() + if err != nil { + return false, errors.Wrapf(err, "unable to GetAllClassMetadata of project %s", info.Project) + } + p.tenant = &ResourceWithClassMetadata{ + classMedata: tcm, + keyword: tenant.Keyword(), + name: tenant.GetName(), + } + + // guest source + guestSource := &ResourceWithClassMetadata{} + disks := info.Disks + var stand *db.SStandaloneAnonResourceBase + // TODO GuestImage + switch { + case len(info.InstanceBackupId) > 0: + obj, err := models.InstanceBackupManager.FetchById(info.InstanceBackupId) + if err != nil { + return false, errors.Wrapf(err, "unable to fetch instanceBackup %s", info.InstanceSnapshotId) + } + stand = &obj.(*models.SInstanceBackup).SStandaloneAnonResourceBase + case len(info.InstanceSnapshotId) > 0: obj, err := models.InstanceSnapshotManager.FetchById(info.InstanceSnapshotId) if err != nil { return false, errors.Wrapf(err, "unable to fetch instanceSnapshot %s", info.InstanceSnapshotId) } - p.instanceSnapshot = obj.(*models.SInstanceSnapshot) - return true, nil - } - if len(info.InstanceBackupId) > 0 { - obj, err := models.InstanceBackupManager.FetchById(info.InstanceBackupId) - if err != nil { - return false, errors.Wrapf(err, "unable to fetch instanceBackup %s", info.InstanceBackupId) - } - p.instanceBackup = obj.(*models.SInstanceBackup) - return true, nil - } - disks := info.Disks - if len(disks) == 0 { - return false, nil - } - switch { + stand = &obj.(*models.SInstanceSnapshot).SStandaloneAnonResourceBase + case len(disks) == 0: case disks[0].ImageId != "": - obj, err := models.CachedimageManager.FetchById(disks[0].ImageId) + obj, err := models.CachedimageManager.GetCachedimageById(context.Background(), disks[0].ImageId) if err != nil { - // 忽略第一次上传到glance镜像后未缓存的记录 - if err == sql.ErrNoRows { - return false, nil - } return false, errors.Wrapf(err, "unable to fetch cachedimage %s", disks[0].ImageId) } - p.cacheImage = obj.(*models.SCachedimage) - return true, nil + stand = &obj.SStandaloneAnonResourceBase + guestSource.keyword = "image" case disks[0].SnapshotId != "": obj, err := models.SnapshotManager.FetchById(disks[0].SnapshotId) if err != nil { return false, errors.Wrapf(err, "unable to fetch snapshot %s", disks[0].SnapshotId) } - p.snapshot = obj.(*models.SSnapshot) - return true, nil + stand = &obj.(*models.SSnapshot).SStandaloneAnonResourceBase case disks[0].BackupId != "": obj, err := models.DiskBackupManager.FetchById(disks[0].BackupId) if err != nil { return false, errors.Wrapf(err, "unable to fetch diskbackup %s", disks[0].BackupId) } - p.backup = obj.(*models.SDiskBackup) + stand = &obj.(*models.SDiskBackup).SStandaloneAnonResourceBase + } + if stand == nil { return true, nil } - return false, nil + cm, err := stand.GetAllClassMetadata() + if err != nil { + return false, errors.Wrapf(err, "unable to GetAllClassMetadata %s", stand.GetId()) + } + guestSource.classMedata = cm + if guestSource.keyword == "" { + guestSource.keyword = stand.Keyword() + } + guestSource.name = stand.GetName() + p.guestSource = guestSource + return true, nil } func (p *ClassMetadataPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(p, u, c) - host := c.Getter().Host() ctx := context.Background() - resourceDesc := "" - var sara *db.SStandaloneAnonResourceBase - switch { - case p.cacheImage != nil: - sara = &p.cacheImage.SStandaloneAnonResourceBase - resourceDesc = fmt.Sprintf("image %s", p.cacheImage.GetName()) - case p.snapshot != nil: - sara = &p.snapshot.SStandaloneAnonResourceBase - resourceDesc = fmt.Sprintf("snapshot %s", p.snapshot.GetName()) - case p.backup != nil: - sara = &p.backup.SStandaloneAnonResourceBase - resourceDesc = fmt.Sprintf("backup %s", p.backup.GetName()) - case p.instanceBackup != nil: - sara = &p.instanceBackup.SStandaloneAnonResourceBase - resourceDesc = fmt.Sprintf("instance backup %s", p.instanceBackup.GetName()) - case p.instanceSnapshot != nil: - sara = &p.instanceSnapshot.SStandaloneAnonResourceBase - resourceDesc = fmt.Sprintf("instance snapshot %s", p.instanceSnapshot.GetName()) - } - ic, err := host.IsInSameClass(ctx, sara) - if err != nil { - return false, nil, errors.Wrap(err, "unable to determine whether they are in a class") - } - if !ic { - h.Exclude(fmt.Sprintf("The host doesn't have the same class metadata as the choosen %s.", resourceDesc)) + for _, resource := range []*ResourceWithClassMetadata{p.tenant, p.guestSource} { + ic, err := db.IsInSameClass(ctx, c.Getter(), resource) + if err != nil { + return false, nil, errors.Wrap(err, "unable to determine whether they are in a class") + } + if !ic { + h.Exclude(fmt.Sprintf("The host doesn't have the same class metadata as the choosen %s.", resource.GetDescription())) + break + } } return h.GetResult() } diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index a719159391..f10fbd1a1a 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -56,6 +56,8 @@ type BaseHostDesc struct { SharedDomains []string `json:"shared_domains"` PendingUsage map[string]interface{} `json:"pending_usage"` + + ClassMetadata map[string]string `json:"class_metadata"` } type baseHostGetter struct { @@ -148,6 +150,10 @@ func (b baseHostGetter) InstanceGroups() map[string]*api.CandidateGroup { return b.h.InstanceGroups } +func (b baseHostGetter) GetAllClassMetadata() (map[string]string, error) { + return b.h.ClassMetadata, nil +} + func (b baseHostGetter) GetFreeGroupCount(groupId string) (int, error) { // Must Be scg, ok := b.h.InstanceGroups[groupId] @@ -333,6 +339,10 @@ func newBaseHostDesc(b *baseBuilder, host *computemodels.SHost) (*BaseHostDesc, return nil, fmt.Errorf("Fill instance group error: %v", err) } + if err := desc.fillClassMetadata(host); err != nil { + return nil, fmt.Errorf("Fill class metadata error: %v", err) + } + if err := desc.fillIpmiInfo(host); err != nil { return nil, fmt.Errorf("Fill ipmi info error: %v", err) } @@ -674,6 +684,15 @@ func (b *BaseHostDesc) fillInstanceGroups(host *computemodels.SHost) error { return nil } +func (b *BaseHostDesc) fillClassMetadata(host *computemodels.SHost) error { + cm, err := host.GetAllClassMetadata() + if err != nil { + return err + } + b.ClassMetadata = cm + return nil +} + func (b *BaseHostDesc) fillIpmiInfo(host *computemodels.SHost) error { info, err := host.GetIpmiInfo() if err != nil { diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 57abe15a71..d9a36ccb8f 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -106,6 +106,8 @@ type CandidatePropertyGetter interface { InstanceGroups() map[string]*api.CandidateGroup GetFreeGroupCount(groupId string) (int, error) + GetAllClassMetadata() (map[string]string, error) + GetIpmiInfo() types.SIPMIInfo GetNics() []*types.SNic From 5bec11dc85709fcf2709cc80accc394b8f01fcbd Mon Sep 17 00:00:00 2001 From: rainzm Date: Wed, 16 Feb 2022 17:31:15 +0800 Subject: [PATCH 11/12] feat: add GetAllClassMetadata for tenantcache --- pkg/cloudcommon/db/tenantcache.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg/cloudcommon/db/tenantcache.go b/pkg/cloudcommon/db/tenantcache.go index b4988518a5..1a09fd789c 100644 --- a/pkg/cloudcommon/db/tenantcache.go +++ b/pkg/cloudcommon/db/tenantcache.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" "runtime/debug" + "strings" "time" "yunion.io/x/jsonutils" @@ -520,3 +521,26 @@ func (tenant *STenant) IsDomain() bool { return false } } + +func (tenant *STenant) objType() string { + if tenant.IsDomain() { + return "domain" + } else { + return "project" + } +} + +func (tenant *STenant) GetAllClassMetadata() (map[string]string, error) { + meta, err := Metadata.rawGetAll(tenant.objType(), tenant.GetId(), nil, CLASS_TAG_PREFIX) + if err != nil { + return nil, errors.Wrap(err, "rawGetAll") + } + ret := make(map[string]string) + for k, v := range meta { + if strings.HasPrefix(k, SYSTEM_ADMIN_PREFIX) { + continue + } + ret[k[len(CLASS_TAG_PREFIX):]] = v + } + return ret, nil +} From 338ecead355dbe644c61d8820b9e687291135625 Mon Sep 17 00:00:00 2001 From: rainzm Date: Wed, 16 Feb 2022 17:44:32 +0800 Subject: [PATCH 12/12] fix(scheduler): add GetAllClassMedata in mock test --- pkg/scheduler/test/mock/core.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/scheduler/test/mock/core.go b/pkg/scheduler/test/mock/core.go index 7330e93a87..e67b2b714e 100644 --- a/pkg/scheduler/test/mock/core.go +++ b/pkg/scheduler/test/mock/core.go @@ -369,6 +369,21 @@ func (mr *MockCandidatePropertyGetterMockRecorder) InstanceGroups() *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceGroups", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).InstanceGroups)) } +// GetAllClassMetadata mocks base method +func (m *MockCandidatePropertyGetter) GetAllClassMetadata() (map[string]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstanceGroups") + ret0, _ := ret[0].(map[string]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAllClassMetadata indicates an expected call of InstanceGroups +func (mr *MockCandidatePropertyGetterMockRecorder) GetAllClassMetadata() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceGroups", reflect.TypeOf((*MockCandidatePropertyGetter)(nil).GetAllClassMetadata)) +} + // IsEmpty mocks base method func (m *MockCandidatePropertyGetter) IsEmpty() bool { m.ctrl.T.Helper()