From 31aec9f969fd9b0388c084030c3803dd2a111bd7 Mon Sep 17 00:00:00 2001 From: Qu Xuan Date: Fri, 8 Jan 2021 20:54:42 +0800 Subject: [PATCH] fix(region): avoid cachedimage never be in sync when it deleted --- pkg/compute/models/cloudimages.go | 17 +++++++++-- pkg/compute/models/cloudregions.go | 13 +++++++- pkg/compute/models/storagecaches.go | 46 ++++++++++++++++++----------- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/pkg/compute/models/cloudimages.go b/pkg/compute/models/cloudimages.go index f36571844e..1aff8074cb 100644 --- a/pkg/compute/models/cloudimages.go +++ b/pkg/compute/models/cloudimages.go @@ -16,6 +16,7 @@ package models import ( "context" + "database/sql" "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -70,7 +71,7 @@ func SyncPublicCloudImages(ctx context.Context, userCred mcclient.TokenCredentia for j := range storagecaches { err = storagecaches[j].CheckCloudimages(ctx, userCred, regions[i].Name, regions[i].Id) if err != nil { - log.Errorf("CheckCloudimages for region %s(%s) storagecache %s error: %v", regions[i].Name, regions[i].Id, storagecaches[j].Name, err) + log.Errorf("SyncSystemImages for region %s(%s) storagecache %s error: %v", regions[i].Name, regions[i].Id, storagecaches[j].Name, err) } } } @@ -94,7 +95,19 @@ func (self *SCloudimage) syncRemove(ctx context.Context, userCred mcclient.Token func (self *SCloudimage) syncWithImage(ctx context.Context, userCred mcclient.TokenCredential, image SCachedimage) error { _cachedImage, err := db.FetchByExternalId(CachedimageManager, image.GetGlobalId()) if err != nil { - return errors.Wrapf(err, "db.FetchByExternalId(%s)", image.GetGlobalId()) + if errors.Cause(err) != sql.ErrNoRows { + return errors.Wrapf(err, "db.FetchByExternalId(%s)", image.GetGlobalId()) + } + image := &image + image.Id = "" + image.IsPublic = true + image.ProjectId = "system" + image.SetModelManager(CachedimageManager, image) + err = CachedimageManager.TableSpec().Insert(ctx, image) + if err != nil { + return errors.Wrapf(err, "Insert cachedimage") + } + return nil } cachedImage := _cachedImage.(*SCachedimage) _, err = db.Update(cachedImage, func() error { diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 7d5f176d18..b318a99324 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -1008,15 +1008,26 @@ func (self *SCloudregion) GetCloudimages() ([]SCloudimage, error) { return images, nil } +func (self *SCloudregion) GetSystemImageCount() (int, error) { + sq := CloudimageManager.Query("external_id").Equals("cloudregion_id", self.Id) + q := CachedimageManager.Query().Equals("image_type", cloudprovider.ImageTypeSystem).In("external_id", sq.SubQuery()) + return q.CountWithError() +} + func (self *SCloudregion) SyncCloudImages(ctx context.Context, userCred mcclient.TokenCredential, refresh bool) error { lockman.LockRawObject(ctx, "cloudimages", self.Id) defer lockman.ReleaseRawObject(ctx, "cloudimages", self.Id) + systemImageCount, err := self.GetSystemImageCount() + if err != nil { + return errors.Wrapf(err, "GetSystemImageCount") + } + dbImages, err := self.GetCloudimages() if err != nil { return errors.Wrapf(err, "GetCloudimages") } - if len(dbImages) > 0 && !refresh { + if len(dbImages) > 0 && systemImageCount > 0 && !refresh { return nil } meta, err := FetchSkuResourcesMeta() diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index a14a6c0cd2..33c7b378d7 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -796,14 +796,14 @@ func (manager *SStoragecacheManager) ListItemExportKeys(ctx context.Context, return q, nil } -func (self *SStoragecache) linkCloudimages(ctx context.Context, regionName, regionId string) error { +func (self *SStoragecache) linkCloudimages(ctx context.Context, regionName, regionId string) (int, error) { cloudimages := CloudimageManager.Query("external_id").Equals("cloudregion_id", regionId).SubQuery() sq := StoragecachedimageManager.Query("cachedimage_id").Equals("storagecache_id", self.Id).SubQuery() - q := CachedimageManager.Query().Equals("image_type", "system").In("external_id", cloudimages).NotIn("id", sq) + q := CachedimageManager.Query().Equals("image_type", cloudprovider.ImageTypeSystem).In("external_id", cloudimages).NotIn("id", sq) images := []SCachedimage{} err := db.FetchModelObjects(CachedimageManager, q, &images) if err != nil { - return errors.Wrapf(err, "db.FetchModelObjects") + return 0, errors.Wrapf(err, "db.FetchModelObjects") } for i := range images { sci := &SStoragecachedimage{} @@ -813,23 +813,20 @@ func (self *SStoragecache) linkCloudimages(ctx context.Context, regionName, regi sci.Status = api.CACHED_IMAGE_STATUS_ACTIVE err = StoragecachedimageManager.TableSpec().Insert(ctx, sci) if err != nil { - return errors.Wrapf(err, "Insert") + return 0, errors.Wrapf(err, "Insert") } } - if len(images) > 0 { - log.Infof("link new %d cloud image for region %s(%s) storagecache %s", len(images), regionName, regionId, self.Name) - } - return nil + return len(images), nil } -func (self *SStoragecache) unlinkCloudimages(ctx context.Context, userCred mcclient.TokenCredential, regionName, regionId string) error { +func (self *SStoragecache) unlinkCloudimages(ctx context.Context, userCred mcclient.TokenCredential, regionName, regionId string) (int, error) { cloudimages := CloudimageManager.Query("external_id").Equals("cloudregion_id", regionId).SubQuery() - sq := CachedimageManager.Query("id").Equals("image_type", "system").NotIn("external_id", cloudimages).SubQuery() + sq := CachedimageManager.Query("id").Equals("image_type", cloudprovider.ImageTypeSystem).NotIn("external_id", cloudimages).SubQuery() q := StoragecachedimageManager.Query().Equals("storagecache_id", self.Id).In("cachedimage_id", sq) scis := []SStoragecachedimage{} err := db.FetchModelObjects(StoragecachedimageManager, q, &scis) if err != nil { - return errors.Wrapf(err, "db.FetchModelObjects") + return 0, errors.Wrapf(err, "db.FetchModelObjects") } for i := range scis { err = scis[i].Delete(ctx, userCred) @@ -837,19 +834,34 @@ func (self *SStoragecache) unlinkCloudimages(ctx context.Context, userCred mccli log.Warningf("detach image %v error: %v", scis[i].GetCachedimage(), err) } } - if len(scis) > 0 { - log.Infof("unlink %d cloud image for region %s(%s) storagecache %s", len(scis), regionName, regionId, self.Name) - } - return nil + return len(scis), nil +} + +func (self *SStoragecache) getSystemImageCount() (int, error) { + sq := StoragecachedimageManager.Query("cachedimage_id").Equals("storagecache_id", self.Id) + q := CachedimageManager.Query().Equals("image_type", cloudprovider.ImageTypeSystem).In("id", sq.SubQuery()) + return q.CountWithError() } func (self *SStoragecache) CheckCloudimages(ctx context.Context, userCred mcclient.TokenCredential, regionName, regionId string) error { lockman.LockRawObject(ctx, "cachedimages", regionId) defer lockman.ReleaseRawObject(ctx, "cachedimages", regionId) - err := self.unlinkCloudimages(ctx, userCred, regionName, regionId) + result := compare.SyncResult{} + + var err error + result.DelCnt, err = self.unlinkCloudimages(ctx, userCred, regionName, regionId) if err != nil { return errors.Wrapf(err, "unlinkCloudimages") } - return self.linkCloudimages(ctx, regionName, regionId) + result.UpdateCnt, err = self.getSystemImageCount() + if err != nil { + log.Errorf("getSystemImageCount error: %v", err) + } + result.AddCnt, err = self.linkCloudimages(ctx, regionName, regionId) + if err != nil { + return errors.Wrapf(err, "linkCloudimages") + } + log.Infof("SycSystemImages for region %s(%s) storagecache %s result: %s", regionName, regionId, self.Name, result.Result()) + return nil }