fix(region): try recover image cache on guest migrate (#21027)

pkg/compute/models/cachedimages.go

@@ -325,6 +325,36 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 	}
 }
 
+func (manager *SCachedimageManager) RecoverCachedImage(ctx context.Context, userCred mcclient.TokenCredential, imgId string) (*SCachedimage, error) {
+	lockman.LockRawObject(ctx, manager.Keyword(), "name")
+	defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name")
+
+	imageCache := SCachedimage{}
+	imageCache.SetModelManager(manager, &imageCache)
+
+	err := manager.RawQuery().Equals("id", imgId).First(&imageCache)
+	if err != nil {
+		return nil, err
+	}
+	diff, err := db.Update(&imageCache, func() error {
+		imageCache.Status = api.CACHED_IMAGE_STATUS_ACTIVE
+		imageCache.LastSync = timeutils.UtcNow()
+		if imageCache.Deleted == true {
+			imageCache.Deleted = false
+			imageCache.DeletedAt = time.Time{}
+			imageCache.RefCount = 0
+			imageCache.UpdateVersion = 0
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	db.OpsLog.LogEvent(&imageCache, db.ACT_UPDATE, diff, userCred)
+	return &imageCache, nil
+}
+
 func (image *SCachedimage) GetStorages() ([]SStorage, error) {
 	sq := StorageManager.Query()
 	storagecacheimageSubq := StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", image.GetId()).SubQuery()

pkg/compute/models/storagecachedimages.go

@@ -339,6 +339,33 @@ func (manager *SStoragecachedimageManager) GetStoragecachedimage(cacheId string,
 	return obj.(*SStoragecachedimage)
 }
 
+func (manager *SStoragecachedimageManager) RecoverStoragecachedImage(
+	ctx context.Context, userCred mcclient.TokenCredential, scId, imgId string,
+) (*SStoragecachedimage, error) {
+	lockman.LockRawObject(ctx, manager.Keyword(), "name")
+	defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name")
+
+	storagecachedImage := SStoragecachedimage{}
+	storagecachedImage.SetModelManager(manager, &storagecachedImage)
+
+	err := manager.RawQuery().Equals("storagecache_id", scId).Equals("cachedimage_id", imgId).First(&storagecachedImage)
+	if err != nil {
+		return nil, err
+	}
+	diff, err := db.Update(&storagecachedImage, func() error {
+		storagecachedImage.Status = api.CACHED_IMAGE_STATUS_ACTIVE
+		if storagecachedImage.Deleted == true {
+			storagecachedImage.Deleted = false
+			storagecachedImage.DeletedAt = time.Time{}
+			storagecachedImage.UpdateVersion = 0
+		}
+		return nil
+	})
+
+	db.OpsLog.LogEvent(&storagecachedImage, db.ACT_UPDATE, diff, userCred)
+	return &storagecachedImage, nil
+}
+
 func (self *SStoragecachedimage) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	_, err := sqlchemy.GetDB().Exec(
 		fmt.Sprintf(

pkg/compute/tasks/guest_live_migrate_task.go

@@ -16,9 +16,11 @@ package tasks
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/httputils"
 	"yunion.io/x/pkg/utils"
@@ -189,6 +191,30 @@ func (task *GuestMigrateTask) SaveScheduleResult(ctx context.Context, obj ISched
 	task.OnStartCacheImages(ctx, guest, nil)
 }
 
+func (task *GuestMigrateTask) tryRecoverImageCache(ctx context.Context, guest *models.SGuest, input *api.CacheImageInput) error {
+	if _, err := models.CachedimageManager.FetchById(input.ImageId); err != nil {
+		if err != sql.ErrNoRows {
+			return err
+		}
+		if _, err := models.CachedimageManager.RecoverCachedImage(ctx, task.UserCred, input.ImageId); err != nil {
+			log.Errorf("failed recache image %s: %s", input.ImageId, err)
+		}
+
+		srcHost, err := guest.GetHost()
+		if err != nil {
+			return err
+		}
+		srcStorageCache := srcHost.GetLocalStoragecache()
+		if scImg := models.StoragecachedimageManager.GetStoragecachedimage(srcStorageCache.Id, input.ImageId); scImg == nil {
+			_, err = models.StoragecachedimageManager.RecoverStoragecachedImage(ctx, task.UserCred, srcStorageCache.Id, input.ImageId)
+			if err != nil {
+				log.Errorf("failed RecoverStoragecachedImage %s:%s %s", srcStorageCache.Id, input.ImageId, err)
+			}
+		}
+	}
+	return nil
+}
+
 func (task *GuestMigrateTask) OnStartCacheImages(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
 	templates, _ := task.Params.GetArray("cache_templates")
 	if len(templates) == 0 {
@@ -210,13 +236,19 @@ func (task *GuestMigrateTask) OnStartCacheImages(ctx context.Context, guest *mod
 			SourceHostId: guest.HostId,
 			ParentTaskId: task.GetTaskId(),
 		}
+		if err := task.tryRecoverImageCache(ctx, guest, &input); err != nil {
+			task.TaskFailed(ctx, guest, jsonutils.NewString(err.Error()))
+			return
+		}
+
 		err := targetStorageCache.StartImageCacheTask(ctx, task.UserCred, input)
 		if err != nil {
 			task.TaskFailed(ctx, guest, jsonutils.NewString(err.Error()))
+			return
 		}
-		return
+	} else {
+		task.OnStartCacheImages(ctx, guest, nil)
 	}
-	task.OnStartCacheImages(ctx, guest, nil)
 }
 
 func (task *GuestMigrateTask) OnStartCacheImagesFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
@@ -237,15 +269,21 @@ func (task *GuestMigrateTask) OnCachedImageComplete(ctx context.Context, guest *
 				Format:       "iso",
 				IsForce:      false,
 				ParentTaskId: task.GetTaskId(),
+				SourceHostId: guest.HostId,
+			}
+			if err := task.tryRecoverImageCache(ctx, guest, &input); err != nil {
+				task.TaskFailed(ctx, guest, jsonutils.NewString(err.Error()))
+				return
 			}
 			err := targetStorageCache.StartImageCacheTask(ctx, task.UserCred, input)
 			if err != nil {
 				task.TaskFailed(ctx, guest, jsonutils.NewString(err.Error()))
+				return
 			}
-			return
 		}
+	} else {
+		task.OnCachedCdromComplete(ctx, guest, nil)
 	}
-	task.OnCachedCdromComplete(ctx, guest, nil)
 }
 
 func (task *GuestMigrateTask) OnCachedCdromComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
@@ -293,6 +331,11 @@ func (task *GuestMigrateTask) OnSrcPrepareComplete(ctx context.Context, guest *m
 	} else {
 		body, err = task.sharedStorageMigrateConf(ctx, guest, targetHost)
 	}
+	if err != nil {
+		task.TaskFailed(ctx, guest, jsonutils.NewString(errors.Wrap(err, "get storage migrate conf").Error()))
+		return
+	}
+
 	if task.isLiveMigrate() {
 		srcDesc, err := data.Get("src_desc")
 		if err != nil {