From 2da0872af3b59087bc573e4725d9daeba48442d6 Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:40:38 +0800 Subject: [PATCH] fix(host): clean image cache check local reference (#20953) --- pkg/hostman/storageman/core.go | 18 ++++---- pkg/hostman/storageman/imagecache_clean.go | 48 +++++++++++++++++++++- pkg/hostman/storageman/storage_base.go | 19 +++++++++ pkg/hostman/storageman/storage_lvm.go | 18 ++++++++ 4 files changed, 93 insertions(+), 10 deletions(-) diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index 77ea7a8326..f3bb4c8f48 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -480,15 +480,15 @@ func CleanImageCachefiles(ctx context.Context, userCred mcclient.TokenCredential for _, imageCacheMan := range storageManager.LVMStorageImagecacheManagers { imageCacheMan.CleanImageCachefiles(ctx) } - for _, imageCacheMan := range storageManager.SharedLVMStorageImagecacheManagers { - imageCacheMan.CleanImageCachefiles(ctx) - } - for _, imageCacheMan := range storageManager.RbdStorageImagecacheManagers { - imageCacheMan.CleanImageCachefiles(ctx) - } - for _, imageCacheMan := range storageManager.SharedFileStorageImagecacheManagers { - imageCacheMan.CleanImageCachefiles(ctx) - } + // for _, imageCacheMan := range storageManager.SharedLVMStorageImagecacheManagers { + // imageCacheMan.CleanImageCachefiles(ctx) + // } + // for _, imageCacheMan := range storageManager.RbdStorageImagecacheManagers { + // imageCacheMan.CleanImageCachefiles(ctx) + // } + // for _, imageCacheMan := range storageManager.SharedFileStorageImagecacheManagers { + // imageCacheMan.CleanImageCachefiles(ctx) + // } } func GatherHostStorageStats() api.SHostPingInput { diff --git a/pkg/hostman/storageman/imagecache_clean.go b/pkg/hostman/storageman/imagecache_clean.go index 2af761dbf6..610ebb47a8 100644 --- a/pkg/hostman/storageman/imagecache_clean.go +++ b/pkg/hostman/storageman/imagecache_clean.go @@ -16,6 +16,7 @@ package storageman import ( "context" + "strings" "time" "yunion.io/x/jsonutils" @@ -28,6 +29,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules/compute" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" baseoptions "yunion.io/x/onecloud/pkg/mcclient/options" + "yunion.io/x/onecloud/pkg/util/qemuimg" ) func cleanImages(ctx context.Context, manager IImageCacheManger, images map[string]IImageCache) (int64, error) { @@ -59,14 +61,53 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri } } - deleteSizeMb := int64(0) + var inUseCacheImageIds = make(map[string]struct{}) + for i := range storageManager.Storages { + storage := storageManager.Storages[i] + if storage.GetStoragecacheId() != manager.GetId() { + continue + } + // load storage disks used image cache + disksPath, err := storage.GetDisksPath() + if err != nil { + log.Errorf("storage %s failed get disksPath: %s", storage.GetPath(), err) + continue + } + for j := range disksPath { + diskPath := disksPath[j] + img, err := qemuimg.NewQemuImage(diskPath) + if err != nil { + log.Errorf("failed NewQemuImage of %s", diskPath) + continue + } + backingChain, err := img.GetBackingChain() + if err != nil { + log.Errorf("disk %s failed get backing chain", diskPath) + continue + } + for _, backingPath := range backingChain { + if strings.HasPrefix(backingPath, manager.GetPath()) { + imageId := strings.Trim(strings.TrimPrefix(backingPath, manager.GetPath()), "/") + inUseCacheImageIds[imageId] = struct{}{} + } + } + } + } + log.Infof("found image caches in use: %v", inUseCacheImageIds) + + deleteSizeMb := int64(0) for imageId, image := range images { if _, ok := storageCachedImages[imageId]; !ok { atime := image.GetDesc().AccessAt if !atime.IsZero() && time.Now().Sub(atime) > time.Duration(options.HostOptions.ImageCacheExpireDays*86400)*time.Second { continue } + if _, ok := inUseCacheImageIds[imageId]; ok { + log.Infof("cached image not found but referenced by disks backing file") + continue + } + log.Infof("cached image %s not found on region, to delete size %dMB ...", imageId, image.GetDesc().SizeMb) // not found on region, clean directly if options.HostOptions.ImageCacheCleanupDryRun { @@ -95,6 +136,11 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri if img.Size == 0 { img.Size = images[imgId].GetDesc().SizeMb * 1024 * 1024 } + if _, ok := inUseCacheImageIds[imgId]; ok { + log.Infof("cached image database reference zero but referenced by disks locally") + continue + } + log.Infof("image reference zero, to delete %s(%s) size %dMB", img.Cachedimage, img.CachedimageId, img.Size/1024/1024) if options.HostOptions.ImageCacheCleanupDryRun { continue diff --git a/pkg/hostman/storageman/storage_base.go b/pkg/hostman/storageman/storage_base.go index b9554ef65f..75aaa6016b 100644 --- a/pkg/hostman/storageman/storage_base.go +++ b/pkg/hostman/storageman/storage_base.go @@ -17,6 +17,7 @@ package storageman import ( "context" "fmt" + "io/ioutil" "path" "strings" "sync" @@ -25,6 +26,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/qemuimgfmt" + "yunion.io/x/pkg/util/regutils" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -116,6 +118,7 @@ type IStorage interface { GetDiskById(diskId string) (IDisk, error) CreateDisk(diskId string) IDisk RemoveDisk(IDisk) + GetDisksPath() ([]string, error) // DeleteDisk(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) @@ -226,6 +229,22 @@ func (s *SBaseStorage) GetZoneId() string { return s.Manager.GetZoneId() } +func (d *SBaseStorage) GetDisksPath() ([]string, error) { + spath := d.GetPath() + files, err := ioutil.ReadDir(spath) + if err != nil { + return nil, err + } + disksPath := make([]string, 0) + for i := range files { + if !files[i].IsDir() && regutils.MatchUUIDExact(files[i].Name()) { + disksPath = append(disksPath, path.Join(spath, files[i].Name())) + } + } + + return disksPath, nil +} + func (s *SBaseStorage) GetCapacityMb() int { return s.GetAvailSizeMb() } diff --git a/pkg/hostman/storageman/storage_lvm.go b/pkg/hostman/storageman/storage_lvm.go index cacca8598f..3a12179ff8 100644 --- a/pkg/hostman/storageman/storage_lvm.go +++ b/pkg/hostman/storageman/storage_lvm.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/qemuimgfmt" + "yunion.io/x/pkg/util/regutils" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -601,6 +602,23 @@ func (s *SLVMStorage) CleanRecycleDiskfiles(ctx context.Context) { log.Infof("SLVMStorage CleanRecycleDiskfiles do nothing!") } +func (d *SLVMStorage) GetDisksPath() ([]string, error) { + spath := d.GetPath() + lvNames, err := lvmutils.GetLvNames(spath) + if err != nil { + return nil, err + } + + disksPath := make([]string, 0) + for _, f := range lvNames { + if regutils.MatchUUIDExact(f) { + disksPath = append(disksPath, path.Join("/dev", spath, f)) + } + } + + return disksPath, nil +} + func ConvertLVMDisk(vgName, lvName string) error { diskPath := path.Join("/dev", vgName, lvName) qemuImg, err := qemuimg.NewQemuImage(diskPath)