diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 6498ca4478..1f732299be 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -128,6 +128,7 @@ func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (* manager := &SGuestManager{} manager.host = host + host.SetIGuestManager(manager) manager.ServersPath = serversPath manager.Servers = new(sync.Map) manager.portsInUse = new(sync.Map) @@ -500,7 +501,11 @@ func (m *SGuestManager) verifyDirtyServers() { func (m *SGuestManager) ClenaupCpuset() { m.Servers.Range(func(k, v interface{}) bool { - guest := v.(*SKVMGuestInstance) + inst := v.(GuestRuntimeInstance) + guest, ok := inst.(*SKVMGuestInstance) + if !ok { + return true + } guest.CleanupCpuset() return true }) @@ -616,7 +621,11 @@ func (m *SGuestManager) LoadServer(sid string) { func (m *SGuestManager) ShutdownServers() { m.Servers.Range(func(k, v interface{}) bool { - guest := v.(*SKVMGuestInstance) + inst := v.(GuestRuntimeInstance) + guest, ok := inst.(*SKVMGuestInstance) + if !ok { + return true + } log.Infof("Start shutdown server %s", guest.GetName()) // scriptStop maybe stuck on guest storage offline @@ -1372,7 +1381,11 @@ func (m *SGuestManager) GetNBDServerFreePort() int { func (m *SGuestManager) GetFreeVncPort() int { vncPorts := make(map[int]struct{}, 0) m.Servers.Range(func(k, v interface{}) bool { - guest := v.(*SKVMGuestInstance) + inst := v.(GuestRuntimeInstance) + guest, ok := inst.(*SKVMGuestInstance) + if !ok { + return true + } inUsePort := guest.GetVncPort() if inUsePort > 0 { vncPorts[inUsePort] = struct{}{} diff --git a/pkg/hostman/guestman/image_deps.go b/pkg/hostman/guestman/image_deps.go new file mode 100644 index 0000000000..560757e2af --- /dev/null +++ b/pkg/hostman/guestman/image_deps.go @@ -0,0 +1,77 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guestman + +import ( + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +func (m *SGuestManager) GetImageDeps(storageType string) []string { + if len(storageType) == 0 { + storageType = api.STORAGE_LOCAL + } + + images := stringutils2.NewSortedStrings(nil) + + m.Servers.Range(func(k, v interface{}) bool { + inst := v.(GuestRuntimeInstance) + imgs := inst.GetDependsImageIds(storageType) + images = images.Append(imgs...) + return true + }) + + return images +} + +func (kvm *SKVMGuestInstance) GetDependsImageIds(storageType string) []string { + images := stringutils2.NewSortedStrings(nil) + for i := range kvm.Desc.Disks { + disk := kvm.Desc.Disks[i] + if len(disk.StorageType) == 0 { + disk.StorageType = api.STORAGE_LOCAL + } + if disk.StorageType != storageType { + continue + } + if disk.TemplateId != "" { + images = images.Append(disk.TemplateId) + } + } + return images +} + +func (pod *sPodGuestInstance) GetDependsImageIds(storageType string) []string { + images := stringutils2.NewSortedStrings(nil) + + for i := range pod.Desc.Containers { + container := pod.Desc.Containers[i] + for j := range container.Spec.VolumeMounts { + volumeMount := container.Spec.VolumeMounts[j] + if volumeMount.Disk != nil { + if volumeMount.Disk.TemplateId != "" { + images = images.Append(volumeMount.Disk.TemplateId) + } + for k := range volumeMount.Disk.PostOverlay { + postOverlay := volumeMount.Disk.PostOverlay[k] + if postOverlay.Image != nil && postOverlay.Image.Id != "" { + images = images.Append(postOverlay.Image.Id) + } + } + } + } + } + return images +} diff --git a/pkg/hostman/guestman/runtime.go b/pkg/hostman/guestman/runtime.go index cc61a559f0..47837e9c99 100644 --- a/pkg/hostman/guestman/runtime.go +++ b/pkg/hostman/guestman/runtime.go @@ -74,6 +74,8 @@ type GuestRuntimeInstance interface { DoSnapshot(ctx context.Context, params *SDiskSnapshot) (jsonutils.JSONObject, error) DeleteSnapshot(ctx context.Context, params *SDeleteDiskSnapshot) (jsonutils.JSONObject, error) OnlineResizeDisk(ctx context.Context, disk storageman.IDisk, sizeMB int64) + + GetDependsImageIds(storageType string) []string } type sBaseGuestInstance struct { diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 0f76603ef0..2dfdaa104e 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -132,6 +132,8 @@ type SHostInfo struct { hasNvidiaGpus *bool hasVastaitechGpus *bool hasCphAmdGpus *bool + + guestManager hostutils.IGuestManager } func (h *SHostInfo) GetContainerDeviceConfigurationFilePath() string { @@ -142,6 +144,14 @@ func (h *SHostInfo) GetContainerCpufreqSimulateConfig() *jsonutils.JSONDict { return h.containerCpufreqSimulateConfig } +func (h *SHostInfo) SetIGuestManager(guestManager hostutils.IGuestManager) { + h.guestManager = guestManager +} + +func (h *SHostInfo) GetIGuestManager() hostutils.IGuestManager { + return h.guestManager +} + func (h *SHostInfo) GetIsolatedDeviceManager() isolated_device.IsolatedDeviceManager { return h.IsolatedDeviceMan } diff --git a/pkg/hostman/hostutils/hostutils.go b/pkg/hostman/hostutils/hostutils.go index c377de279d..0caab62a35 100644 --- a/pkg/hostman/hostutils/hostutils.go +++ b/pkg/hostman/hostutils/hostutils.go @@ -61,6 +61,10 @@ type SContainerCpufreqSimulateConfig struct { ScalingAvailableGovernors string `json:"scaling_available_governors"` } +type IGuestManager interface { + GetImageDeps(storageType string) []string +} + type IHost interface { GetZoneId() string GetHostId() string @@ -102,6 +106,9 @@ type IHost interface { OnCatalogChanged(catalog mcclient.KeystoneServiceCatalogV3) OnHostFilesChanged(hostfiles []computeapi.SHostFile) error + + SetIGuestManager(guestman IGuestManager) + GetIGuestManager() IGuestManager } func GetComputeSession(ctx context.Context) *mcclient.ClientSession { diff --git a/pkg/hostman/storageman/imagecache_clean.go b/pkg/hostman/storageman/imagecache_clean.go index 610ebb47a8..0723375949 100644 --- a/pkg/hostman/storageman/imagecache_clean.go +++ b/pkg/hostman/storageman/imagecache_clean.go @@ -30,6 +30,7 @@ import ( modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" baseoptions "yunion.io/x/onecloud/pkg/mcclient/options" "yunion.io/x/onecloud/pkg/util/qemuimg" + "yunion.io/x/onecloud/pkg/util/stringutils2" ) func cleanImages(ctx context.Context, manager IImageCacheManger, images map[string]IImageCache) (int64, error) { @@ -61,39 +62,8 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri } } - var inUseCacheImageIds = make(map[string]struct{}) - for i := range storageManager.Storages { - storage := storageManager.Storages[i] - if storage.GetStoragecacheId() != manager.GetId() { - continue - } - // load storage disks used image cache - disksPath, err := storage.GetDisksPath() - if err != nil { - log.Errorf("storage %s failed get disksPath: %s", storage.GetPath(), err) - continue - } + inUseCacheImageIds := findCachedImagesInUse(manager) - for j := range disksPath { - diskPath := disksPath[j] - img, err := qemuimg.NewQemuImage(diskPath) - if err != nil { - log.Errorf("failed NewQemuImage of %s", diskPath) - continue - } - backingChain, err := img.GetBackingChain() - if err != nil { - log.Errorf("disk %s failed get backing chain", diskPath) - continue - } - for _, backingPath := range backingChain { - if strings.HasPrefix(backingPath, manager.GetPath()) { - imageId := strings.Trim(strings.TrimPrefix(backingPath, manager.GetPath()), "/") - inUseCacheImageIds[imageId] = struct{}{} - } - } - } - } log.Infof("found image caches in use: %v", inUseCacheImageIds) deleteSizeMb := int64(0) @@ -103,7 +73,8 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri if !atime.IsZero() && time.Now().Sub(atime) > time.Duration(options.HostOptions.ImageCacheExpireDays*86400)*time.Second { continue } - if _, ok := inUseCacheImageIds[imageId]; ok { + + if inUseCacheImageIds.Contains(imageId) { log.Infof("cached image not found but referenced by disks backing file") continue } @@ -136,7 +107,8 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri if img.Size == 0 { img.Size = images[imgId].GetDesc().SizeMb * 1024 * 1024 } - if _, ok := inUseCacheImageIds[imgId]; ok { + + if inUseCacheImageIds.Contains(imgId) { log.Infof("cached image database reference zero but referenced by disks locally") continue } @@ -155,3 +127,51 @@ func cleanImages(ctx context.Context, manager IImageCacheManger, images map[stri return deleteSizeMb, nil } + +func findCachedImagesInUse(manager IImageCacheManger) stringutils2.SSortedStrings { + imageIds := stringutils2.NewSortedStrings(nil) + for _, imageId := range findQumuImagesInUse(manager) { + imageIds = imageIds.Append(imageId) + } + for _, imageId := range storageManager.host.GetIGuestManager().GetImageDeps(manager.GetStorageType()) { + imageIds = imageIds.Append(imageId) + } + return imageIds +} + +func findQumuImagesInUse(manager IImageCacheManger) stringutils2.SSortedStrings { + imageIds := stringutils2.NewSortedStrings(nil) + for i := range storageManager.Storages { + storage := storageManager.Storages[i] + if storage.GetStoragecacheId() != manager.GetId() { + continue + } + // load storage disks used image cache + disksPath, err := storage.GetDisksPath() + if err != nil { + log.Errorf("storage %s failed get disksPath: %s", storage.GetPath(), err) + continue + } + + for j := range disksPath { + diskPath := disksPath[j] + img, err := qemuimg.NewQemuImage(diskPath) + if err != nil { + log.Errorf("failed NewQemuImage of %s", diskPath) + continue + } + backingChain, err := img.GetBackingChain() + if err != nil { + log.Errorf("disk %s failed get backing chain", diskPath) + continue + } + for _, backingPath := range backingChain { + if strings.HasPrefix(backingPath, manager.GetPath()) { + imageId := strings.Trim(strings.TrimPrefix(backingPath, manager.GetPath()), "/") + imageIds = imageIds.Append(imageId) + } + } + } + } + return imageIds +} diff --git a/pkg/hostman/storageman/imagecachemanager_base.go b/pkg/hostman/storageman/imagecachemanager_base.go index e38ec94956..d17ae2e8e6 100644 --- a/pkg/hostman/storageman/imagecachemanager_base.go +++ b/pkg/hostman/storageman/imagecachemanager_base.go @@ -52,6 +52,7 @@ type IImageCacheManger interface { Lvmlockd() bool IsLocal() bool + GetStorageType() string // for diskhandler PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error) diff --git a/pkg/hostman/storageman/imagecachemanager_local.go b/pkg/hostman/storageman/imagecachemanager_local.go index a8cdf133f8..aeaa3839e6 100644 --- a/pkg/hostman/storageman/imagecachemanager_local.go +++ b/pkg/hostman/storageman/imagecachemanager_local.go @@ -67,6 +67,13 @@ func (c *SLocalImageCacheManager) IsLocal() bool { return true } +func (c *SLocalImageCacheManager) GetStorageType() string { + if c.storage == nil { + return api.STORAGE_LOCAL + } + return c.storage.StorageType() +} + func (c *SLocalImageCacheManager) loadCache(ctx context.Context) { if len(c.cachePath) == 0 { return @@ -123,6 +130,11 @@ func (c *SLocalImageCacheManager) DeleteImageCache(ctx context.Context, data int return nil, hostutils.ParamsError } + cachedImagesInUser := findCachedImagesInUse(c) + if cachedImagesInUser.Contains(input.ImageId) { + return nil, httperrors.NewResourceBusyError("image cache is in use") + } + return nil, c.RemoveImage(ctx, input.ImageId) } diff --git a/pkg/hostman/storageman/imagecachemanager_lvm.go b/pkg/hostman/storageman/imagecachemanager_lvm.go index 5a8702b6df..e213bfea0f 100644 --- a/pkg/hostman/storageman/imagecachemanager_lvm.go +++ b/pkg/hostman/storageman/imagecachemanager_lvm.go @@ -61,6 +61,10 @@ func (c *SLVMImageCacheManager) IsLocal() bool { return c.storage.IsLocal() } +func (c *SLVMImageCacheManager) GetStorageType() string { + return c.storage.StorageType() +} + func (c *SLVMImageCacheManager) Lvmlockd() bool { return c.lvmlockd } @@ -159,6 +163,11 @@ func (c *SLVMImageCacheManager) DeleteImageCache(ctx context.Context, data inter return nil, hostutils.ParamsError } + cachedImagesInUser := findCachedImagesInUse(c) + if cachedImagesInUser.Contains(input.ImageId) { + return nil, httperrors.NewResourceBusyError("image cache is in use") + } + if input.DeactivateImage != nil && *input.DeactivateImage { return nil, c.DeactiveImageCacahe(ctx, input.ImageId) } else { diff --git a/pkg/hostman/storageman/imagecachemanager_rbd.go b/pkg/hostman/storageman/imagecachemanager_rbd.go index 3362009424..3f589d7de4 100644 --- a/pkg/hostman/storageman/imagecachemanager_rbd.go +++ b/pkg/hostman/storageman/imagecachemanager_rbd.go @@ -114,6 +114,10 @@ func (c *SRbdImageCacheManager) IsLocal() bool { return false } +func (c *SRbdImageCacheManager) GetStorageType() string { + return c.storage.StorageType() +} + func (c *SRbdImageCacheManager) GetPath() string { return c.Pool } @@ -156,6 +160,11 @@ func (c *SRbdImageCacheManager) DeleteImageCache(ctx context.Context, data inter return nil, hostutils.ParamsError } + cachedImagesInUser := findCachedImagesInUse(c) + if cachedImagesInUser.Contains(input.ImageId) { + return nil, httperrors.NewResourceBusyError("image cache is in use") + } + return nil, c.RemoveImage(ctx, input.ImageId) }