fix: host clean image with regards of container image deps (#22955)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

pkg/hostman/guestman/guestman.go

@@ -128,6 +128,7 @@ func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (*
 
 	manager := &SGuestManager{}
 	manager.host = host
+	host.SetIGuestManager(manager)
 	manager.ServersPath = serversPath
 	manager.Servers = new(sync.Map)
 	manager.portsInUse = new(sync.Map)
@@ -500,7 +501,11 @@ func (m *SGuestManager) verifyDirtyServers() {
 
 func (m *SGuestManager) ClenaupCpuset() {
 	m.Servers.Range(func(k, v interface{}) bool {
-		guest := v.(*SKVMGuestInstance)
+		inst := v.(GuestRuntimeInstance)
+		guest, ok := inst.(*SKVMGuestInstance)
+		if !ok {
+			return true
+		}
 		guest.CleanupCpuset()
 		return true
 	})
@@ -616,7 +621,11 @@ func (m *SGuestManager) LoadServer(sid string) {
 
 func (m *SGuestManager) ShutdownServers() {
 	m.Servers.Range(func(k, v interface{}) bool {
-		guest := v.(*SKVMGuestInstance)
+		inst := v.(GuestRuntimeInstance)
+		guest, ok := inst.(*SKVMGuestInstance)
+		if !ok {
+			return true
+		}
 		log.Infof("Start shutdown server %s", guest.GetName())
 
 		// scriptStop maybe stuck on guest storage offline
@@ -1372,7 +1381,11 @@ func (m *SGuestManager) GetNBDServerFreePort() int {
 func (m *SGuestManager) GetFreeVncPort() int {
 	vncPorts := make(map[int]struct{}, 0)
 	m.Servers.Range(func(k, v interface{}) bool {
-		guest := v.(*SKVMGuestInstance)
+		inst := v.(GuestRuntimeInstance)
+		guest, ok := inst.(*SKVMGuestInstance)
+		if !ok {
+			return true
+		}
 		inUsePort := guest.GetVncPort()
 		if inUsePort > 0 {
 			vncPorts[inUsePort] = struct{}{}

pkg/hostman/guestman/image_deps.go

@@ -0,0 +1,77 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package guestman
+
+import (
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+func (m *SGuestManager) GetImageDeps(storageType string) []string {
+	if len(storageType) == 0 {
+		storageType = api.STORAGE_LOCAL
+	}
+
+	images := stringutils2.NewSortedStrings(nil)
+
+	m.Servers.Range(func(k, v interface{}) bool {
+		inst := v.(GuestRuntimeInstance)
+		imgs := inst.GetDependsImageIds(storageType)
+		images = images.Append(imgs...)
+		return true
+	})
+
+	return images
+}
+
+func (kvm *SKVMGuestInstance) GetDependsImageIds(storageType string) []string {
+	images := stringutils2.NewSortedStrings(nil)
+	for i := range kvm.Desc.Disks {
+		disk := kvm.Desc.Disks[i]
+		if len(disk.StorageType) == 0 {
+			disk.StorageType = api.STORAGE_LOCAL
+		}
+		if disk.StorageType != storageType {
+			continue
+		}
+		if disk.TemplateId != "" {
+			images = images.Append(disk.TemplateId)
+		}
+	}
+	return images
+}
+
+func (pod *sPodGuestInstance) GetDependsImageIds(storageType string) []string {
+	images := stringutils2.NewSortedStrings(nil)
+
+	for i := range pod.Desc.Containers {
+		container := pod.Desc.Containers[i]
+		for j := range container.Spec.VolumeMounts {
+			volumeMount := container.Spec.VolumeMounts[j]
+			if volumeMount.Disk != nil {
+				if volumeMount.Disk.TemplateId != "" {
+					images = images.Append(volumeMount.Disk.TemplateId)
+				}
+				for k := range volumeMount.Disk.PostOverlay {
+					postOverlay := volumeMount.Disk.PostOverlay[k]
+					if postOverlay.Image != nil && postOverlay.Image.Id != "" {
+						images = images.Append(postOverlay.Image.Id)
+					}
+				}
+			}
+		}
+	}
+	return images
+}

pkg/hostman/guestman/runtime.go

@@ -74,6 +74,8 @@ type GuestRuntimeInstance interface {
 	DoSnapshot(ctx context.Context, params *SDiskSnapshot) (jsonutils.JSONObject, error)
 	DeleteSnapshot(ctx context.Context, params *SDeleteDiskSnapshot) (jsonutils.JSONObject, error)
 	OnlineResizeDisk(ctx context.Context, disk storageman.IDisk, sizeMB int64)
+
+	GetDependsImageIds(storageType string) []string
 }
 
 type sBaseGuestInstance struct {

pkg/hostman/hostinfo/hostinfo.go

@@ -132,6 +132,8 @@ type SHostInfo struct {
 	hasNvidiaGpus                  *bool
 	hasVastaitechGpus              *bool
 	hasCphAmdGpus                  *bool
+
+	guestManager hostutils.IGuestManager
 }
 
 func (h *SHostInfo) GetContainerDeviceConfigurationFilePath() string {
@@ -142,6 +144,14 @@ func (h *SHostInfo) GetContainerCpufreqSimulateConfig() *jsonutils.JSONDict {
 	return h.containerCpufreqSimulateConfig
 }
 
+func (h *SHostInfo) SetIGuestManager(guestManager hostutils.IGuestManager) {
+	h.guestManager = guestManager
+}
+
+func (h *SHostInfo) GetIGuestManager() hostutils.IGuestManager {
+	return h.guestManager
+}
+
 func (h *SHostInfo) GetIsolatedDeviceManager() isolated_device.IsolatedDeviceManager {
 	return h.IsolatedDeviceMan
 }

pkg/hostman/hostutils/hostutils.go

@@ -61,6 +61,10 @@ type SContainerCpufreqSimulateConfig struct {
 	ScalingAvailableGovernors string `json:"scaling_available_governors"`
 }
 
+type IGuestManager interface {
+	GetImageDeps(storageType string) []string
+}
+
 type IHost interface {
 	GetZoneId() string
 	GetHostId() string
@@ -102,6 +106,9 @@ type IHost interface {
 	OnCatalogChanged(catalog mcclient.KeystoneServiceCatalogV3)
 
 	OnHostFilesChanged(hostfiles []computeapi.SHostFile) error
+
+	SetIGuestManager(guestman IGuestManager)
+	GetIGuestManager() IGuestManager
 }
 
 func GetComputeSession(ctx context.Context) *mcclient.ClientSession {

pkg/hostman/storageman/imagecache_clean.go

@@ -30,6 +30,7 @@ import (
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
 	baseoptions "yunion.io/x/onecloud/pkg/mcclient/options"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 func cleanImages(ctx context.Context, manager IImageCacheManger, images map[string]IImageCache) (int64, error) {
@@ -61,39 +62,8 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri
 		}
 	}
 
-	var inUseCacheImageIds = make(map[string]struct{})
-	for i := range storageManager.Storages {
-		storage := storageManager.Storages[i]
-		if storage.GetStoragecacheId() != manager.GetId() {
-			continue
-		}
-		// load storage disks used image cache
-		disksPath, err := storage.GetDisksPath()
-		if err != nil {
-			log.Errorf("storage %s failed get disksPath: %s", storage.GetPath(), err)
-			continue
-		}
+	inUseCacheImageIds := findCachedImagesInUse(manager)
 
-		for j := range disksPath {
-			diskPath := disksPath[j]
-			img, err := qemuimg.NewQemuImage(diskPath)
-			if err != nil {
-				log.Errorf("failed NewQemuImage of %s", diskPath)
-				continue
-			}
-			backingChain, err := img.GetBackingChain()
-			if err != nil {
-				log.Errorf("disk %s failed get backing chain", diskPath)
-				continue
-			}
-			for _, backingPath := range backingChain {
-				if strings.HasPrefix(backingPath, manager.GetPath()) {
-					imageId := strings.Trim(strings.TrimPrefix(backingPath, manager.GetPath()), "/")
-					inUseCacheImageIds[imageId] = struct{}{}
-				}
-			}
-		}
-	}
 	log.Infof("found image caches in use: %v", inUseCacheImageIds)
 
 	deleteSizeMb := int64(0)
@@ -103,7 +73,8 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri
 			if !atime.IsZero() && time.Now().Sub(atime) > time.Duration(options.HostOptions.ImageCacheExpireDays*86400)*time.Second {
 				continue
 			}
-			if _, ok := inUseCacheImageIds[imageId]; ok {
+
+			if inUseCacheImageIds.Contains(imageId) {
 				log.Infof("cached image not found but referenced by disks backing file")
 				continue
 			}
@@ -136,7 +107,8 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri
 			if img.Size == 0 {
 				img.Size = images[imgId].GetDesc().SizeMb * 1024 * 1024
 			}
-			if _, ok := inUseCacheImageIds[imgId]; ok {
+
+			if inUseCacheImageIds.Contains(imgId) {
 				log.Infof("cached image database reference zero but referenced by disks locally")
 				continue
 			}
@@ -155,3 +127,51 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri
 
 	return deleteSizeMb, nil
 }
+
+func findCachedImagesInUse(manager IImageCacheManger) stringutils2.SSortedStrings {
+	imageIds := stringutils2.NewSortedStrings(nil)
+	for _, imageId := range findQumuImagesInUse(manager) {
+		imageIds = imageIds.Append(imageId)
+	}
+	for _, imageId := range storageManager.host.GetIGuestManager().GetImageDeps(manager.GetStorageType()) {
+		imageIds = imageIds.Append(imageId)
+	}
+	return imageIds
+}
+
+func findQumuImagesInUse(manager IImageCacheManger) stringutils2.SSortedStrings {
+	imageIds := stringutils2.NewSortedStrings(nil)
+	for i := range storageManager.Storages {
+		storage := storageManager.Storages[i]
+		if storage.GetStoragecacheId() != manager.GetId() {
+			continue
+		}
+		// load storage disks used image cache
+		disksPath, err := storage.GetDisksPath()
+		if err != nil {
+			log.Errorf("storage %s failed get disksPath: %s", storage.GetPath(), err)
+			continue
+		}
+
+		for j := range disksPath {
+			diskPath := disksPath[j]
+			img, err := qemuimg.NewQemuImage(diskPath)
+			if err != nil {
+				log.Errorf("failed NewQemuImage of %s", diskPath)
+				continue
+			}
+			backingChain, err := img.GetBackingChain()
+			if err != nil {
+				log.Errorf("disk %s failed get backing chain", diskPath)
+				continue
+			}
+			for _, backingPath := range backingChain {
+				if strings.HasPrefix(backingPath, manager.GetPath()) {
+					imageId := strings.Trim(strings.TrimPrefix(backingPath, manager.GetPath()), "/")
+					imageIds = imageIds.Append(imageId)
+				}
+			}
+		}
+	}
+	return imageIds
+}

pkg/hostman/storageman/imagecachemanager_base.go

@@ -52,6 +52,7 @@ type IImageCacheManger interface {
 	Lvmlockd() bool
 
 	IsLocal() bool
+	GetStorageType() string
 
 	// for diskhandler
 	PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error)

pkg/hostman/storageman/imagecachemanager_local.go

@@ -67,6 +67,13 @@ func (c *SLocalImageCacheManager) IsLocal() bool {
 	return true
 }
 
+func (c *SLocalImageCacheManager) GetStorageType() string {
+	if c.storage == nil {
+		return api.STORAGE_LOCAL
+	}
+	return c.storage.StorageType()
+}
+
 func (c *SLocalImageCacheManager) loadCache(ctx context.Context) {
 	if len(c.cachePath) == 0 {
 		return
@@ -123,6 +130,11 @@ func (c *SLocalImageCacheManager) DeleteImageCache(ctx context.Context, data int
 		return nil, hostutils.ParamsError
 	}
 
+	cachedImagesInUser := findCachedImagesInUse(c)
+	if cachedImagesInUser.Contains(input.ImageId) {
+		return nil, httperrors.NewResourceBusyError("image cache is in use")
+	}
+
 	return nil, c.RemoveImage(ctx, input.ImageId)
 }
 

pkg/hostman/storageman/imagecachemanager_lvm.go

@@ -61,6 +61,10 @@ func (c *SLVMImageCacheManager) IsLocal() bool {
 	return c.storage.IsLocal()
 }
 
+func (c *SLVMImageCacheManager) GetStorageType() string {
+	return c.storage.StorageType()
+}
+
 func (c *SLVMImageCacheManager) Lvmlockd() bool {
 	return c.lvmlockd
 }
@@ -159,6 +163,11 @@ func (c *SLVMImageCacheManager) DeleteImageCache(ctx context.Context, data inter
 		return nil, hostutils.ParamsError
 	}
 
+	cachedImagesInUser := findCachedImagesInUse(c)
+	if cachedImagesInUser.Contains(input.ImageId) {
+		return nil, httperrors.NewResourceBusyError("image cache is in use")
+	}
+
 	if input.DeactivateImage != nil && *input.DeactivateImage {
 		return nil, c.DeactiveImageCacahe(ctx, input.ImageId)
 	} else {

pkg/hostman/storageman/imagecachemanager_rbd.go

@@ -114,6 +114,10 @@ func (c *SRbdImageCacheManager) IsLocal() bool {
 	return false
 }
 
+func (c *SRbdImageCacheManager) GetStorageType() string {
+	return c.storage.StorageType()
+}
+
 func (c *SRbdImageCacheManager) GetPath() string {
 	return c.Pool
 }
@@ -156,6 +160,11 @@ func (c *SRbdImageCacheManager) DeleteImageCache(ctx context.Context, data inter
 		return nil, hostutils.ParamsError
 	}
 
+	cachedImagesInUser := findCachedImagesInUse(c)
+	if cachedImagesInUser.Contains(input.ImageId) {
+		return nil, httperrors.NewResourceBusyError("image cache is in use")
+	}
+
 	return nil, c.RemoveImage(ctx, input.ImageId)
 }