From dc1c6c754cd372b8504947927339f707405e8cb3 Mon Sep 17 00:00:00 2001 From: Qu Xuan Date: Tue, 1 Sep 2020 20:16:38 +0800 Subject: [PATCH] optimized cachedimages sync --- pkg/apis/compute/cachedimage.go | 2 +- pkg/apis/compute/cachedimages_const.go | 1 + pkg/apis/compute/input.go | 2 +- pkg/cloudprovider/resources.go | 3 +- pkg/compute/models/cachedimages.go | 143 +++++++++++++----- pkg/compute/models/cloudproviders.go | 3 +- pkg/compute/models/initdb.go | 1 + pkg/compute/models/storagecachedimages.go | 10 +- pkg/compute/models/storagecaches.go | 15 +- .../tasks/cloud_provider_delete_task.go | 2 +- pkg/mcclient/modules/mod_cachedimages.go | 2 +- pkg/multicloud/aliyun/image.go | 4 +- pkg/multicloud/aws/image.go | 4 +- pkg/multicloud/azure/image.go | 4 +- pkg/multicloud/ctyun/image.go | 4 +- pkg/multicloud/esxi/image.go | 6 +- pkg/multicloud/esxi/template.go | 6 +- pkg/multicloud/google/image.go | 4 +- pkg/multicloud/google/storagecache.go | 2 +- pkg/multicloud/huawei/image.go | 4 +- pkg/multicloud/image_base.go | 25 +++ pkg/multicloud/openstack/image.go | 34 +++-- pkg/multicloud/openstack/shell/image.go | 4 +- pkg/multicloud/openstack/storagecache.go | 2 +- pkg/multicloud/qcloud/image.go | 4 +- pkg/multicloud/qcloud/storagecache.go | 2 +- pkg/multicloud/ucloud/host.go | 4 +- pkg/multicloud/ucloud/image.go | 4 +- pkg/multicloud/ucloud/storagecache.go | 2 +- pkg/multicloud/zstack/image.go | 7 +- pkg/multicloud/zstack/storagecache.go | 4 +- 31 files changed, 227 insertions(+), 87 deletions(-) create mode 100644 pkg/multicloud/image_base.go diff --git a/pkg/apis/compute/cachedimage.go b/pkg/apis/compute/cachedimage.go index 70ada4b2bc..e10e7e65c1 100644 --- a/pkg/apis/compute/cachedimage.go +++ b/pkg/apis/compute/cachedimage.go @@ -27,7 +27,7 @@ type CachedImageUncacheImageInput struct { } type CachedimageDetails struct { - apis.StandaloneResourceDetails + apis.SharableVirtualResourceDetails SCachedimage diff --git a/pkg/apis/compute/cachedimages_const.go b/pkg/apis/compute/cachedimages_const.go index 06d6fbbaef..4c2628f4e1 100644 --- a/pkg/apis/compute/cachedimages_const.go +++ b/pkg/apis/compute/cachedimages_const.go @@ -18,6 +18,7 @@ const ( CACHED_IMAGE_STATUS_INIT = "init" CACHED_IMAGE_STATUS_SAVING = "saving" CACHED_IMAGE_STATUS_CACHING = "caching" + CACHED_IMAGE_STATUS_ACTIVE = "active" CACHED_IMAGE_STATUS_READY = "ready" CACHED_IMAGE_STATUS_DELETING = "deleting" CACHED_IMAGE_STATUS_CACHE_FAILED = "cache_fail" diff --git a/pkg/apis/compute/input.go b/pkg/apis/compute/input.go index c47f64eb1b..032311df97 100644 --- a/pkg/apis/compute/input.go +++ b/pkg/apis/compute/input.go @@ -49,7 +49,7 @@ type KeypairListInput struct { } type CachedimageListInput struct { - apis.StandaloneResourceListInput + apis.SharableVirtualResourceListInput apis.ExternalizedResourceBaseListInput ManagedResourceListInput diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index e7dd26db86..df106c1670 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -156,7 +156,7 @@ type ICloudZone interface { } type ICloudImage interface { - ICloudResource + IVirtualResource Delete(ctx context.Context) error GetIStoragecache() ICloudStoragecache @@ -173,6 +173,7 @@ type ICloudImage interface { GetImageFormat() string GetCreatedAt() time.Time UEFI() bool + GetPublicScope() rbacutils.TRbacScope } type ICloudStoragecache interface { diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index 2a71d964bc..82fcde3d9c 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -37,11 +37,12 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/stringutils2" ) type SCachedimageManager struct { - db.SStandaloneResourceBaseManager + db.SSharableVirtualResourceBaseManager db.SExternalizedResourceBaseManager } @@ -49,7 +50,7 @@ var CachedimageManager *SCachedimageManager func init() { CachedimageManager = &SCachedimageManager{ - SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager( + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( SCachedimage{}, "cachedimages_tbl", "cachedimage", @@ -60,7 +61,7 @@ func init() { } type SCachedimage struct { - db.SStandaloneResourceBase + db.SSharableVirtualResourceBase db.SExternalizedResourceBase // 镜像大小单位: Byte @@ -100,10 +101,10 @@ func (self *SCachedimage) ValidateDeleteCondition(ctx context.Context) error { if cnt > 0 { return httperrors.NewNotEmptyError("The image has been cached on storages") } - if self.GetStatus() == "active" && !self.isReferenceSessionExpire() { + if self.GetStatus() == api.CACHED_IMAGE_STATUS_ACTIVE && !self.isReferenceSessionExpire() { return httperrors.NewConflictError("the image reference session has not been expired!") } - return self.SStandaloneResourceBase.ValidateDeleteCondition(ctx) + return self.SSharableVirtualResourceBase.ValidateDeleteCondition(ctx) } func (self *SCachedimage) isReferenceSessionExpire() bool { @@ -194,32 +195,44 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred)) defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred)) - imgId, _ := info.GetString("id") - if len(imgId) == 0 { + img := struct { + Id string + Size int64 + Name string + Status string + IsPublic bool + ProjectId string `json:"tenant_id"` + PublicScope string + }{} + err := info.Unmarshal(&img) + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal %s", info.String()) + } + if len(img.Id) == 0 { return nil, fmt.Errorf("invalid image info") } + if len(img.Name) == 0 { + img.Name = img.Id + } imageCache := SCachedimage{} imageCache.SetModelManager(manager, &imageCache) - size, _ := info.Int("size") - name, _ := info.GetString("name") - if len(name) == 0 { - name = imgId - } - - name, err := db.GenerateName(manager, nil, name) + img.Name, err = db.GenerateName(manager, nil, img.Name) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "db.GenerateName(%s)", img.Name) } - err = manager.RawQuery().Equals("id", imgId).First(&imageCache) + err = manager.RawQuery().Equals("id", img.Id).First(&imageCache) if err != nil { if err == sql.ErrNoRows { // insert - imageCache.Id = imgId - imageCache.Name = name - imageCache.Size = size + imageCache.Id = img.Id + imageCache.Name = img.Name + imageCache.Size = img.Size imageCache.Info = info + imageCache.Status = img.Status + imageCache.IsPublic = img.IsPublic + imageCache.PublicScope = img.PublicScope imageCache.LastSync = timeutils.UtcNow() err = manager.TableSpec().Insert(ctx, &imageCache) @@ -230,13 +243,16 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us return &imageCache, nil } else { - log.Errorf("fetching image cache (%s) failed: %s", imgId, err) + log.Errorf("fetching image cache (%s) failed: %s", img.Id, err) return nil, err } } else { // update diff, err := db.Update(&imageCache, func() error { - imageCache.Size = size + imageCache.Size = img.Size imageCache.Info = info + imageCache.Status = img.Status + imageCache.IsPublic = img.IsPublic + imageCache.PublicScope = img.PublicScope imageCache.LastSync = timeutils.UtcNow() if imageCache.Deleted == true { imageCache.Deleted = false @@ -323,16 +339,15 @@ func (manager *SCachedimageManager) FetchCustomizeColumns( isList bool, ) []api.CachedimageDetails { rows := make([]api.CachedimageDetails, len(objs)) - stdRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + virtRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) for i := range rows { ci := objs[i].(*SCachedimage) rows[i] = api.CachedimageDetails{ - StandaloneResourceDetails: stdRows[i], - Status: ci.GetStatus(), - OsType: ci.GetOSType(), - OsDistribution: ci.GetOSDistribution(), - OsVersion: ci.GetOSVersion(), - Hypervisor: ci.GetHypervisor(), + SharableVirtualResourceDetails: virtRows[i], + OsType: ci.GetOSType(), + OsDistribution: ci.GetOSDistribution(), + OsVersion: ci.GetOSVersion(), + Hypervisor: ci.GetHypervisor(), } rows[i].CachedCount, _ = ci.getStoragecacheCount() } @@ -372,7 +387,7 @@ func (self *SCachedimage) PerformUncacheImage(ctx context.Context, userCred mccl } func (self *SCachedimage) addRefCount() { - if self.GetStatus() != "active" { + if self.GetStatus() != api.CACHED_IMAGE_STATUS_ACTIVE { return } _, err := db.Update(self, func() error { @@ -455,12 +470,17 @@ func (self *SCachedimage) canDeleteLastCache() bool { return false } -func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) error { +func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error { diff, err := db.UpdateWithLock(ctx, self, func() error { self.Name = image.GetName() self.Size = image.GetSizeByte() self.ExternalId = image.GetGlobalId() self.ImageType = image.GetImageType() + self.PublicScope = string(image.GetPublicScope()) + self.Status = image.GetStatus() + if image.GetPublicScope() == rbacutils.ScopeSystem { + self.IsPublic = true + } self.UEFI = tristate.NewFromBool(image.UEFI()) sImage := cloudprovider.CloudImage2Image(image) self.Info = jsonutils.Marshal(&sImage) @@ -468,10 +488,12 @@ func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mccli return nil }) db.OpsLog.LogSyncUpdate(self, diff, userCred) + + SyncCloudProject(userCred, self, ownerId, image, managerId) return err } -func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) (*SCachedimage, error) { +func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) (*SCachedimage, error) { lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred)) defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred)) @@ -491,12 +513,21 @@ func (manager *SCachedimageManager) newFromCloudImage(ctx context.Context, userC cachedImage.LastSync = time.Now().UTC() cachedImage.ImageType = image.GetImageType() cachedImage.ExternalId = image.GetGlobalId() + cachedImage.Status = image.GetStatus() + cachedImage.PublicScope = string(image.GetPublicScope()) + switch image.GetPublicScope() { + case rbacutils.ScopeNone: + default: + cachedImage.IsPublic = true + } err = manager.TableSpec().Insert(ctx, &cachedImage) if err != nil { return nil, err } + SyncCloudProject(userCred, &cachedImage, ownerId, image, managerId) + return &cachedImage, nil } @@ -529,7 +560,7 @@ func (image *SCachedimage) requestRefreshExternalImage(ctx context.Context, user log.Errorf("iCache.GetIImageById fail %s", err) return nil, err } - err = image.syncWithCloudImage(ctx, userCred, iImage) + err = image.syncWithCloudImage(ctx, userCred, nil, iImage, "") if err != nil { log.Errorf("image.syncWithCloudImage fail %s", err) return nil, err @@ -632,6 +663,10 @@ func (manager *SCachedimageManager) ListItemFilter( query api.CachedimageListInput, ) (*sqlchemy.SQuery, error) { var err error + q, err = manager.SSharableBaseResourceManager.ListItemFilter(ctx, q, userCred, query.SharableResourceBaseListInput) + if err != nil { + return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter") + } q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery { cachedImages := CachedimageManager.Query().SubQuery() @@ -647,9 +682,9 @@ func (manager *SCachedimageManager) ListItemFilter( return nil, errors.Wrap(err, "managedResourceFilterByAccount") } - q, err = manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneResourceListInput) + q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput) if err != nil { - return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter") + return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter") } q, err = manager.SExternalizedResourceBaseManager.ListItemFilter(ctx, q, userCred, query.ExternalizedResourceBaseListInput) @@ -704,9 +739,9 @@ func (manager *SCachedimageManager) OrderByExtraFields( ) (*sqlchemy.SQuery, error) { var err error - q, err = manager.SStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneResourceListInput) + q, err = manager.SSharableVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.SharableVirtualResourceListInput) if err != nil { - return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.OrderByExtraFields") + return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.OrderByExtraFields") } return q, nil @@ -715,10 +750,44 @@ func (manager *SCachedimageManager) OrderByExtraFields( func (manager *SCachedimageManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { var err error - q, err = manager.SStandaloneResourceBaseManager.QueryDistinctExtraField(q, field) + q, err = manager.SSharableVirtualResourceBaseManager.QueryDistinctExtraField(q, field) if err == nil { return q, nil } return q, httperrors.ErrNotFound } + +func (manager *SCachedimageManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) { + q, err := manager.SSharableVirtualResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) + if err != nil { + return nil, errors.Wrapf(err, "SSharableVirtualResourceBaseManager.L.ListItemExportKeys") + } + return q, nil +} + +func (manager *SCachedimageManager) InitializeData() error { + images := []SCachedimage{} + q := manager.Query().IsNullOrEmpty("tenant_id") + err := db.FetchModelObjects(manager, q, &images) + if err != nil { + return errors.Wrapf(err, "db.FetchModelObjects") + } + for i := range images { + _, err := db.Update(&images[i], func() error { + images[i].IsPublic = true + images[i].PublicScope = string(rbacutils.ScopeSystem) + images[i].ProjectId = "system" + if len(images[i].ExternalId) > 0 { + images[i].Status = api.CACHED_IMAGE_STATUS_ACTIVE + } else { + images[i].Status = images[i].GetStatus() + } + return nil + }) + if err != nil { + return errors.Wrapf(err, "db.Update(%s)", images[i].Id) + } + } + return nil +} diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 27f57aaa02..7b0cf1cb84 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -1397,8 +1397,7 @@ func (self *SCloudprovider) StartCloudproviderDeleteTask(ctx context.Context, us params := jsonutils.NewDict() task, err := taskman.TaskManager.NewTask(ctx, "CloudProviderDeleteTask", self, userCred, params, parentTaskId, "", nil) if err != nil { - log.Errorf("%s", err) - return err + return errors.Wrapf(err, "NewTask") } self.SetStatus(userCred, api.CLOUD_PROVIDER_START_DELETE, "StartCloudproviderDeleteTask") task.ScheduleRun(nil) diff --git a/pkg/compute/models/initdb.go b/pkg/compute/models/initdb.go index 45501c9049..a81d7e75a4 100644 --- a/pkg/compute/models/initdb.go +++ b/pkg/compute/models/initdb.go @@ -60,6 +60,7 @@ func InitDB() error { ScheduledTaskActivityManager, ExternalProjectManager, + CachedimageManager, } { err := manager.InitializeData() if err != nil { diff --git a/pkg/compute/models/storagecachedimages.go b/pkg/compute/models/storagecachedimages.go index 7314e23b92..17278bb5d6 100644 --- a/pkg/compute/models/storagecachedimages.go +++ b/pkg/compute/models/storagecachedimages.go @@ -431,17 +431,17 @@ func (self *SStoragecachedimage) syncRemoveCloudImage(ctx context.Context, userC return nil } -func (self *SStoragecachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage) error { +func (self *SStoragecachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error { cachedImage := self.GetCachedimage() if len(cachedImage.ExternalId) > 0 { self.SetStatus(userCred, image.GetStatus(), "") - return cachedImage.syncWithCloudImage(ctx, userCred, image) + return cachedImage.syncWithCloudImage(ctx, userCred, ownerId, image, managerId) } else { return nil } } -func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, image cloudprovider.ICloudImage, cache *SStoragecache) error { +func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, cache *SStoragecache) error { var cachedImage *SCachedimage imgObj, err := db.FetchByExternalId(CachedimageManager, image.GetGlobalId()) if err != nil { @@ -464,7 +464,7 @@ func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context } if cachedImage == nil { // no such image - cachedImage, err = CachedimageManager.newFromCloudImage(ctx, userCred, image) + cachedImage, err = CachedimageManager.newFromCloudImage(ctx, userCred, ownerId, image, cache.ManagerId) if err != nil { log.Errorf("CachedimageManager.newFromCloudImage fail %s", err) return err @@ -474,7 +474,7 @@ func (manager *SStoragecachedimageManager) newFromCloudImage(ctx context.Context cachedImage = imgObj.(*SCachedimage) } if len(cachedImage.ExternalId) > 0 { - cachedImage.syncWithCloudImage(ctx, userCred, image) + cachedImage.syncWithCloudImage(ctx, userCred, ownerId, image, cache.ManagerId) } scimg := manager.Register(ctx, userCred, cache.GetId(), cachedImage.GetId(), image.GetStatus()) if scimg == nil { diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index f120273770..a37132897f 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -610,6 +610,13 @@ func (cache *SStoragecache) SyncCloudImages( return syncResult } + var syncOwnerId mcclient.IIdentityProvider + + provider := cache.GetCloudprovider() + if provider != nil { + syncOwnerId = provider.GetOwnerId() + } + removed := make([]SStoragecachedimage, 0) commondb := make([]SStoragecachedimage, 0) commonext := make([]cloudprovider.ICloudImage, 0) @@ -631,7 +638,7 @@ func (cache *SStoragecache) SyncCloudImages( } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudImage(ctx, userCred, commonext[i]) + err = commondb[i].syncWithCloudImage(ctx, userCred, syncOwnerId, commonext[i], cache.ManagerId) if err != nil { syncResult.UpdateError(err) } else { @@ -639,7 +646,7 @@ func (cache *SStoragecache) SyncCloudImages( } } for i := 0; i < len(added); i += 1 { - err = StoragecachedimageManager.newFromCloudImage(ctx, userCred, added[i], cache) + err = StoragecachedimageManager.newFromCloudImage(ctx, userCred, syncOwnerId, added[i], cache) if err != nil { syncResult.AddError(err) } else { @@ -662,7 +669,7 @@ func (self *SStoragecache) IsReachCapacityLimit(imageId string) bool { log.Debugf("image %s is not a customized image, no need to cache", imageId) return false } - cachedImages := self.getCachedImageList(nil, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_READY}) + cachedImages := self.getCachedImageList(nil, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_ACTIVE}) for i := range cachedImages { if cachedImages[i].Id == imageId { // already cached @@ -675,7 +682,7 @@ func (self *SStoragecache) IsReachCapacityLimit(imageId string) bool { } func (self *SStoragecache) StartRelinquishLeastUsedCachedImageTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, parentTaskId string) error { - cachedImages := self.getCachedImageList([]string{imageId}, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_READY}) + cachedImages := self.getCachedImageList([]string{imageId}, cloudprovider.CachedImageTypeCustomized, []string{api.CACHED_IMAGE_STATUS_ACTIVE}) leastUsedIdx := -1 leastRefCount := -1 for i := range cachedImages { diff --git a/pkg/compute/tasks/cloud_provider_delete_task.go b/pkg/compute/tasks/cloud_provider_delete_task.go index 47740eb927..242d92821f 100644 --- a/pkg/compute/tasks/cloud_provider_delete_task.go +++ b/pkg/compute/tasks/cloud_provider_delete_task.go @@ -42,7 +42,7 @@ func (self *CloudProviderDeleteTask) OnInit(ctx context.Context, obj db.IStandal err := provider.RealDelete(ctx, self.UserCred) if err != nil { provider.SetStatus(self.UserCred, api.CLOUD_PROVIDER_DELETE_FAILED, "StartDiskCloudproviderTask") - self.SetStageFailed(ctx, jsonutils.Marshal(err)) + self.SetStageFailed(ctx, jsonutils.NewString(err.Error())) logclient.AddActionLogWithStartable(self, provider, logclient.ACT_DELETE, err, self.UserCred, false) return } diff --git a/pkg/mcclient/modules/mod_cachedimages.go b/pkg/mcclient/modules/mod_cachedimages.go index 506386a679..8bf762f202 100644 --- a/pkg/mcclient/modules/mod_cachedimages.go +++ b/pkg/mcclient/modules/mod_cachedimages.go @@ -26,7 +26,7 @@ func init() { "OS_Type", "OS_Distribution", "OS_version", "Hypervisor", "Host_count", "Status", "Ref_Count", "cached_count", "image_type", - "External_Id", + "External_Id", "Is_Public", "Public_Scope", "Tenant", "Tenant_Id", }, []string{}) diff --git a/pkg/multicloud/aliyun/image.go b/pkg/multicloud/aliyun/image.go index 8d0013a8a2..3e23361fd2 100644 --- a/pkg/multicloud/aliyun/image.go +++ b/pkg/multicloud/aliyun/image.go @@ -28,6 +28,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/imagetools" ) @@ -57,6 +58,7 @@ const ( ) type SImage struct { + multicloud.SImageBase storageCache *SStoragecache Architecture string @@ -129,7 +131,7 @@ func (self *SImage) GetStatus() string { case ImageStatusCreating: return api.CACHED_IMAGE_STATUS_SAVING case ImageStatusAvailable: - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE case ImageStatusUnAvailable: return api.CACHED_IMAGE_STATUS_CACHE_FAILED case ImageStatusCreateFailed: diff --git a/pkg/multicloud/aws/image.go b/pkg/multicloud/aws/image.go index 6204355b20..3b7f572130 100644 --- a/pkg/multicloud/aws/image.go +++ b/pkg/multicloud/aws/image.go @@ -28,6 +28,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" ) type ImageStatusType string @@ -74,6 +75,7 @@ type RootDevice struct { } type SImage struct { + multicloud.SImageBase storageCache *SStoragecache Architecture string @@ -185,7 +187,7 @@ func (self *SImage) GetStatus() string { case ImageStatusCreating: return api.CACHED_IMAGE_STATUS_CACHING case ImageStatusAvailable: - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE case ImageStatusCreateFailed: return api.CACHED_IMAGE_STATUS_CACHE_FAILED default: diff --git a/pkg/multicloud/azure/image.go b/pkg/multicloud/azure/image.go index 8bf9c528b6..63223dd1ad 100644 --- a/pkg/multicloud/azure/image.go +++ b/pkg/multicloud/azure/image.go @@ -27,6 +27,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" ) type ImageStatusType string @@ -80,6 +81,7 @@ type ImageProperties struct { } type SImage struct { + multicloud.SImageBase storageCache *SStoragecache Properties ImageProperties `json:"properties,omitempty"` @@ -130,7 +132,7 @@ func (self *SImage) GetStatus() string { case "created": return api.CACHED_IMAGE_STATUS_CACHING case "Succeeded": - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE default: log.Errorf("Unknow image status: %s", self.Properties.ProvisioningState) return api.CACHED_IMAGE_STATUS_CACHE_FAILED diff --git a/pkg/multicloud/ctyun/image.go b/pkg/multicloud/ctyun/image.go index afec3385a2..2267a8dc27 100644 --- a/pkg/multicloud/ctyun/image.go +++ b/pkg/multicloud/ctyun/image.go @@ -24,6 +24,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/imagetools" ) @@ -35,6 +36,7 @@ const ( // http://ctyun-api-url/apiproxy/v3/order/getImages type SImage struct { + multicloud.SImageBase storageCache *SStoragecache ID string `json:"id"` @@ -66,7 +68,7 @@ func (self *SImage) GetGlobalId() string { } func (self *SImage) GetStatus() string { - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE } func (self *SImage) Refresh() error { diff --git a/pkg/multicloud/esxi/image.go b/pkg/multicloud/esxi/image.go index 738eb62246..218fad1d82 100644 --- a/pkg/multicloud/esxi/image.go +++ b/pkg/multicloud/esxi/image.go @@ -26,9 +26,11 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" ) type SImage struct { + multicloud.SImageBase cache *SDatastoreImageCache filename string size int64 @@ -74,12 +76,12 @@ func (self *SImage) GetStatus() string { if err != nil { return api.CACHED_IMAGE_STATUS_CACHE_FAILED } - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE } func (self *SImage) GetImageStatus() string { status := self.GetStatus() - if status == api.CACHED_IMAGE_STATUS_READY { + if status == api.CACHED_IMAGE_STATUS_ACTIVE { return cloudprovider.IMAGE_STATUS_ACTIVE } return cloudprovider.IMAGE_STATUS_DELETED diff --git a/pkg/multicloud/esxi/template.go b/pkg/multicloud/esxi/template.go index ba3be4a244..717957cc16 100644 --- a/pkg/multicloud/esxi/template.go +++ b/pkg/multicloud/esxi/template.go @@ -24,9 +24,11 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" ) type SVMTemplate struct { + multicloud.SImageBase cache *SDatastoreImageCache vm *SVirtualMachine uuid string @@ -65,7 +67,7 @@ func (t *SVMTemplate) GetStatus() string { host := ihost.(*SHost) _, err := host.GetTemplateVMById(t.uuid) if err == nil { - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE } if errors.Cause(err) != cloudprovider.ErrNotFound { log.Errorf("fail to find templatevm %q: %v", t.uuid, err) @@ -123,7 +125,7 @@ func (t *SVMTemplate) GetImageType() string { func (t *SVMTemplate) GetImageStatus() string { status := t.GetStatus() - if status == api.CACHED_IMAGE_STATUS_READY { + if status == api.CACHED_IMAGE_STATUS_ACTIVE { return cloudprovider.IMAGE_STATUS_ACTIVE } return cloudprovider.IMAGE_STATUS_DELETED diff --git a/pkg/multicloud/google/image.go b/pkg/multicloud/google/image.go index 7004119271..b248bdd1aa 100644 --- a/pkg/multicloud/google/image.go +++ b/pkg/multicloud/google/image.go @@ -25,6 +25,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/imagetools" ) @@ -39,6 +40,7 @@ type SDeprecated struct { } type SImage struct { + multicloud.SImageBase storagecache *SStoragecache SResourceBase @@ -120,7 +122,7 @@ func (image *SImage) GetMinRamSizeMb() int { func (image *SImage) GetStatus() string { switch image.Status { case "READY": - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE case "FAILED": return api.CACHED_IMAGE_STATUS_CACHE_FAILED case "PENDING": diff --git a/pkg/multicloud/google/storagecache.go b/pkg/multicloud/google/storagecache.go index 0c7ae53389..07dca38a10 100644 --- a/pkg/multicloud/google/storagecache.go +++ b/pkg/multicloud/google/storagecache.go @@ -102,7 +102,7 @@ func (cache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.T } else { status := _image.GetStatus() log.Debugf("UploadImage: Image external ID %s exists, status %s", image.ExternalId, status) - if status == api.CACHED_IMAGE_STATUS_READY { + if status == api.CACHED_IMAGE_STATUS_ACTIVE { return image.ExternalId, nil } err = cache.region.Delete(image.ExternalId) diff --git a/pkg/multicloud/huawei/image.go b/pkg/multicloud/huawei/image.go index 8c832ee793..b9e98639fe 100644 --- a/pkg/multicloud/huawei/image.go +++ b/pkg/multicloud/huawei/image.go @@ -26,6 +26,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/imagetools" ) @@ -50,6 +51,7 @@ const ( // https://support.huaweicloud.com/api-ims/zh-cn_topic_0020091565.html type SImage struct { + multicloud.SImageBase storageCache *SStoragecache Schema string `json:"schema"` @@ -112,7 +114,7 @@ func (self *SImage) GetStatus() string { case ImageStatusQueued: return api.CACHED_IMAGE_STATUS_CACHING case ImageStatusActive: - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE case ImageStatusKilled: return api.CACHED_IMAGE_STATUS_CACHE_FAILED default: diff --git a/pkg/multicloud/image_base.go b/pkg/multicloud/image_base.go new file mode 100644 index 0000000000..8908665be1 --- /dev/null +++ b/pkg/multicloud/image_base.go @@ -0,0 +1,25 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multicloud + +import "yunion.io/x/onecloud/pkg/util/rbacutils" + +type SImageBase struct { + SVirtualResourceBase +} + +func (self *SImageBase) GetPublicScope() rbacutils.TRbacScope { + return rbacutils.ScopeSystem +} diff --git a/pkg/multicloud/openstack/image.go b/pkg/multicloud/openstack/image.go index 1ed90dbe99..a2f7e3ae3b 100644 --- a/pkg/multicloud/openstack/image.go +++ b/pkg/multicloud/openstack/image.go @@ -29,8 +29,10 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/imagetools" "yunion.io/x/onecloud/pkg/util/qemuimg" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) const ( @@ -46,6 +48,7 @@ const ( ) type SImage struct { + multicloud.SImageBase storageCache *SStoragecache Status string @@ -157,7 +160,7 @@ func (image *SImage) GetStatus() string { case QUEUED, SAVING, UPLOADING, IMPORTING: return api.CACHED_IMAGE_STATUS_CACHING case ACTIVE: - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE case DELETED, DEACTIVATED, PENDING_DELETE, KILLED: return api.CACHED_IMAGE_STATUS_CACHE_FAILED default: @@ -192,6 +195,19 @@ func (image *SImage) GetImageType() string { return cloudprovider.CachedImageTypeSystem } +func (image *SImage) GetPublicScope() rbacutils.TRbacScope { + switch image.Visibility { + case "private": + return rbacutils.ScopeNone + default: + return rbacutils.ScopeSystem + } +} + +func (image *SImage) GetProjectId() string { + return image.Owner +} + func (image *SImage) GetSizeByte() int64 { return int64(image.Size) } @@ -246,17 +262,17 @@ func (image *SImage) GetCreatedAt() time.Time { } func (region *SRegion) GetImage(imageId string) (*SImage, error) { - images, err := region.GetImages("", "", imageId) + resource := "/v2/images/" + imageId + resp, err := region.imageGet(resource) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "imageGet(%s)", resource) } - if len(images) == 0 { - return nil, cloudprovider.ErrNotFound + image := &SImage{} + err = resp.Unmarshal(image) + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") } - if len(images) > 1 { - return nil, cloudprovider.ErrDuplicateId - } - return &images[0], nil + return image, nil } func (image *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache { diff --git a/pkg/multicloud/openstack/shell/image.go b/pkg/multicloud/openstack/shell/image.go index 2b28c3286c..f82cda7643 100644 --- a/pkg/multicloud/openstack/shell/image.go +++ b/pkg/multicloud/openstack/shell/image.go @@ -43,11 +43,11 @@ func init() { } shellutils.R(&ImageOptions{}, "image-show", "Show image", func(cli *openstack.SRegion, args *ImageOptions) error { - image, err := cli.GetImages("", "", args.ID) + image, err := cli.GetImage(args.ID) if err != nil { return err } - printObject(image[0]) + printObject(image) return nil }) diff --git a/pkg/multicloud/openstack/storagecache.go b/pkg/multicloud/openstack/storagecache.go index 12441939fe..84694714e4 100644 --- a/pkg/multicloud/openstack/storagecache.go +++ b/pkg/multicloud/openstack/storagecache.go @@ -158,7 +158,7 @@ func (cache *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.T img.storageCache = cache - return img.Id, cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, 15*time.Second, 3600*time.Second) + return img.Id, cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_ACTIVE, 15*time.Second, 3600*time.Second) } func (cache *SStoragecache) CreateIImage(snapshoutId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { diff --git a/pkg/multicloud/qcloud/image.go b/pkg/multicloud/qcloud/image.go index 800b53f65c..4f4237ac7e 100644 --- a/pkg/multicloud/qcloud/image.go +++ b/pkg/multicloud/qcloud/image.go @@ -27,6 +27,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/imagetools" ) @@ -42,6 +43,7 @@ const ( ) type SImage struct { + multicloud.SImageBase storageCache *SStoragecache ImageId string // 镜像ID @@ -140,7 +142,7 @@ func (self *SImage) GetStatus() string { case ImageStatusCreating, ImageStatusSycing, ImageStatusImporting: return api.CACHED_IMAGE_STATUS_CACHING case ImageStatusNormal, ImageStatusUsing: - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE default: return api.CACHED_IMAGE_STATUS_CACHE_FAILED } diff --git a/pkg/multicloud/qcloud/storagecache.go b/pkg/multicloud/qcloud/storagecache.go index c71d1dda18..e2c133653e 100644 --- a/pkg/multicloud/qcloud/storagecache.go +++ b/pkg/multicloud/qcloud/storagecache.go @@ -223,7 +223,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To if err != nil { return "", err } - err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, 15*time.Second, 3600*time.Second) + err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_ACTIVE, 15*time.Second, 3600*time.Second) if err != nil { return "", err } diff --git a/pkg/multicloud/ucloud/host.go b/pkg/multicloud/ucloud/host.go index 12c9a61ec1..799195b75d 100644 --- a/pkg/multicloud/ucloud/host.go +++ b/pkg/multicloud/ucloud/host.go @@ -258,8 +258,8 @@ func (self *SHost) _createVM(name, imgId string, sysDisk cloudprovider.SDiskInfo log.Errorf("GetImage %s fail %s", imgId, err) return "", err } - if img.GetStatus() != api.CACHED_IMAGE_STATUS_READY { - log.Errorf("image %s status %s, expect %s", imgId, img.GetStatus(), api.CACHED_IMAGE_STATUS_READY) + if img.GetStatus() != api.CACHED_IMAGE_STATUS_ACTIVE { + log.Errorf("image %s status %s, expect %s", imgId, img.GetStatus(), api.CACHED_IMAGE_STATUS_ACTIVE) return "", fmt.Errorf("image not ready") } diff --git a/pkg/multicloud/ucloud/image.go b/pkg/multicloud/ucloud/image.go index d4afa8e7fe..e5ab347b4a 100644 --- a/pkg/multicloud/ucloud/image.go +++ b/pkg/multicloud/ucloud/image.go @@ -23,10 +23,12 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/imagetools" ) type SImage struct { + multicloud.SImageBase storageCache *SStoragecache Zone string `json:"Zone"` @@ -65,7 +67,7 @@ func (self *SImage) GetGlobalId() string { func (self *SImage) GetStatus() string { switch self.State { case "Available": - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE case "Making": return api.CACHED_IMAGE_STATUS_CACHING case "Unavailable": diff --git a/pkg/multicloud/ucloud/storagecache.go b/pkg/multicloud/ucloud/storagecache.go index cdcaef867b..39a3a01a99 100644 --- a/pkg/multicloud/ucloud/storagecache.go +++ b/pkg/multicloud/ucloud/storagecache.go @@ -125,7 +125,7 @@ func (self *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.To if err != nil { log.Errorf("GetImageStatus error %s", err) } - if img.GetStatus() == api.CACHED_IMAGE_STATUS_READY && !isForce { + if img.GetStatus() == api.CACHED_IMAGE_STATUS_ACTIVE && !isForce { return image.ExternalId, nil } } else { diff --git a/pkg/multicloud/zstack/image.go b/pkg/multicloud/zstack/image.go index c3d8d7fad4..c5eb728e15 100644 --- a/pkg/multicloud/zstack/image.go +++ b/pkg/multicloud/zstack/image.go @@ -23,13 +23,13 @@ import ( "sort" "time" - "github.com/pkg/errors" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/imagetools" "yunion.io/x/onecloud/pkg/util/multipart" @@ -45,6 +45,7 @@ type SBackupStorageRef struct { } type SImage struct { + multicloud.SImageBase storageCache *SStoragecache BackupStorageRefs []SBackupStorageRef `json:"backupStorageRefs"` @@ -117,7 +118,7 @@ func (image *SImage) GetStatus() string { } switch image.Status { case "Ready": - return api.CACHED_IMAGE_STATUS_READY + return api.CACHED_IMAGE_STATUS_ACTIVE case "Downloading": return api.CACHED_IMAGE_STATUS_CACHING case "Deleted": diff --git a/pkg/multicloud/zstack/storagecache.go b/pkg/multicloud/zstack/storagecache.go index fb9829ee70..56c8de8d32 100644 --- a/pkg/multicloud/zstack/storagecache.go +++ b/pkg/multicloud/zstack/storagecache.go @@ -105,7 +105,7 @@ func (scache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient. log.Errorf("GetImageStatus error %s", err) } status := img.GetStatus() - if api.CACHED_IMAGE_STATUS_READY == status && !isForce { + if api.CACHED_IMAGE_STATUS_ACTIVE == status && !isForce { return image.ExternalId, nil } log.Debugf("image %s status %s", image.ExternalId, status) @@ -131,7 +131,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To return "", err } img.storageCache = self - err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, time.Second*5, time.Minute*20) //windows镜像转换比较慢，等待时间稍微设长一些 + err = cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_ACTIVE, time.Second*5, time.Minute*20) //windows镜像转换比较慢，等待时间稍微设长一些 if err != nil { log.Errorf("waitting for image %s(%s) status ready timeout", img.Name, img.UUID) }