opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-08-31 01:35:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feature: automatically clean obsolete images from host iamge cache (#20217)

Browse Source 
Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
Jian Qiu
2024-05-09 12:11:26 +08:00
committed by GitHub
parent 44eb35e30e
commit 209019e3f4
23 changed files with 460 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/compute/storagecachedimage.go
+2
View File
@@ -21,6 +21,8 @@ import (
)
type StoragecachedimageDetails struct {
SStoragecachedimage
apis.JointResourceBaseDetails
StoragecacheResourceInfo
pkg/cloudcommon/agent/agent.go
+1 -1
View File
@@ -276,7 +276,7 @@ func (agent *SBaseAgent) createOrUpdateBaremetalAgent(session *mcclient.ClientSe
return errors.Error("agent not support storagecache_id, region might not be up-to-date")
}
}
agent.CacheManager = storageman.NewLocalImageCacheManager(agent.IAgent(), agent.CachePath, storageCacheId)
agent.CacheManager = storageman.NewLocalImageCacheManager(agent.IAgent(), agent.CachePath, storageCacheId, nil)
return nil
}
pkg/hostman/host_services.go
+7 -3
View File
@@ -130,9 +130,13 @@ func (host *SHostService) RunService() {
},
)
cronManager.AddJobEveryFewDays(
"CleanRecycleDiskFiles", 1, 3, 0, 0, storageman.CleanRecycleDiskfiles, false)
cronManager.Start()
{
cronManager.AddJobEveryFewDays(
"CleanRecycleDiskFiles", 1, 3, 0, 0, storageman.CleanRecycleDiskfiles, false)
cronManager.AddJobEveryFewDays(
"CleanImageCachefiles", 1, 3, 0, 0, storageman.CleanImageCachefiles, options.HostOptions.ImageCacheCleanupOnStartup)
cronManager.Start()
}
close(guestChan)
app_common.ServeForeverWithCleanup(app, &options.HostOptions.BaseOptions, func() {
pkg/hostman/hostinfo/hostinfo.go
+1 -1
View File
@@ -1974,7 +1974,7 @@ func (h *SHostInfo) initStoragesInternal(hoststorages []jsonutils.JSONObject) {
}
if storagetype == api.STORAGE_LVM {
// lvm set storage image cache info
storageManager.InitLVMStorageImageCache(storagecacheId, mountPoint)
storageManager.InitLVMStorageImageCache(storagecacheId, mountPoint, storage)
}
} else {
// XXX hack: storage type baremetal is a converted host，reserve storage
pkg/hostman/options/options.go
+5
View File
@@ -48,6 +48,11 @@ type SHostBaseOptions struct {
Ext4LargefileSizeGb int `default:"4096" help:"Use largefile options when the ext4 fs greater than this size"`
Ext4HugefileSizeGb int `default:"512" help:"Use huge options when the ext4 fs greater than this size"`
ImageCacheExpireDays int `help:"Image cache expire duration in days" default:"30"`
ImageCacheCleanupPercentage int `help:"The cleanup threshold ratio of image cache size v.s. total storage size" default:"12"`
ImageCacheCleanupOnStartup bool `help:"Cleanup image cache on host startup" default:"false"`
ImageCacheCleanupDryRun bool `help:"Dry run cleanup image cache" default:"true"`
}
type SHostOptions struct {
pkg/hostman/storageman/core.go
+44 -13
View File
@@ -17,7 +17,7 @@ package storageman
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
"strings"
"time"
@@ -210,7 +210,7 @@ func (s *SStorageManager) initLocalStorageImagecache() error {
}
}
if len(cachePath) > 0 {
s.LocalStorageImagecacheManager = NewLocalImageCacheManager(s, cachePath, "")
s.LocalStorageImagecacheManager = NewLocalImageCacheManager(s, cachePath, "", nil)
return nil
} else {
return fmt.Errorf("Cannot allocate image cache storage")
@@ -300,7 +300,17 @@ func (s *SStorageManager) GetDiskByPath(diskPath string) (IDisk, error) {
func (s *SStorageManager) GetTotalCapacity() int {
var capa = 0
for _, s := range s.Storages {
capa += s.GetCapacity()
capa += s.GetCapacityMb()
}
return capa
}
func (s *SStorageManager) GetTotalLocalCapacity() int {
var capa = 0
for _, s := range s.Storages {
if _, ok := s.(*SLocalStorage); ok {
capa += s.GetCapacityMb()
}
}
return capa
}
@@ -331,7 +341,7 @@ func (s *SStorageManager) NewSharedStorageInstance(mountPoint, storageType strin
func (s *SStorageManager) InitSharedStorageImageCache(storageType, storagecacheId, imagecachePath string, storage IStorage) {
if utils.IsInStringArray(storageType, api.SHARED_FILE_STORAGE) {
s.InitSharedFileStorageImagecache(storagecacheId, imagecachePath)
s.InitSharedFileStorageImagecache(storagecacheId, imagecachePath, storage)
} else if storageType == api.STORAGE_RBD {
if rbdStorageCache := s.GetStoragecacheById(storagecacheId); rbdStorageCache == nil {
s.AddRbdStorageImagecache(imagecachePath, storage, storagecacheId)
@@ -343,7 +353,7 @@ func (s *SStorageManager) InitSharedStorageImageCache(storageType, storagecacheI
}
}
func (s *SStorageManager) InitLVMStorageImageCache(storagecacheId, vg string) {
func (s *SStorageManager) InitLVMStorageImageCache(storagecacheId, vg string, storage IStorage) {
if len(storagecacheId) == 0 {
return
}
@@ -351,11 +361,11 @@ func (s *SStorageManager) InitLVMStorageImageCache(storagecacheId, vg string) {
s.LVMStorageImagecacheManagers = map[string]IImageCacheManger{}
}
if _, ok := s.LVMStorageImagecacheManagers[storagecacheId]; !ok {
s.LVMStorageImagecacheManagers[storagecacheId] = NewLVMImageCacheManager(s, vg, storagecacheId, false)
s.LVMStorageImagecacheManagers[storagecacheId] = NewLVMImageCacheManager(s, vg, storagecacheId, storage, false)
}
}
func (s *SStorageManager) InitSharedFileStorageImagecache(storagecacheId, path string) {
func (s *SStorageManager) InitSharedFileStorageImagecache(storagecacheId, path string, storage IStorage) {
if len(path) == 0 {
return
}
@@ -363,7 +373,7 @@ func (s *SStorageManager) InitSharedFileStorageImagecache(storagecacheId, path s
s.SharedFileStorageImagecacheManagers = map[string]IImageCacheManger{}
}
if _, ok := s.SharedFileStorageImagecacheManagers[storagecacheId]; !ok {
s.SharedFileStorageImagecacheManagers[storagecacheId] = NewLocalImageCacheManager(s, path, storagecacheId)
s.SharedFileStorageImagecacheManagers[storagecacheId] = NewLocalImageCacheManager(s, path, storagecacheId, storage)
}
}
@@ -372,7 +382,7 @@ func (s *SStorageManager) AddSharedLVMStorageImagecache(imagecachePath string, s
s.SharedLVMStorageImagecacheManagers = map[string]IImageCacheManger{}
}
if _, ok := s.SharedLVMStorageImagecacheManagers[storagecacheId]; !ok {
imagecache := NewLVMImageCacheManager(s, imagecachePath, storagecacheId, storage.Lvmlockd())
imagecache := NewLVMImageCacheManager(s, imagecachePath, storagecacheId, storage, storage.Lvmlockd())
s.SharedLVMStorageImagecacheManagers[storagecacheId] = imagecache
}
}
@@ -422,7 +432,7 @@ func cleanDailyFiles(storagePath, subDir string, keepDay int) {
// before mark should be deleted
markTime := timeutils.UtcNow().Add(time.Hour * 24 * -1 * time.Duration(keepDay))
files, err := ioutil.ReadDir(recycleDir)
files, err := os.ReadDir(recycleDir)
if err != nil {
log.Errorln(err)
return
@@ -454,9 +464,30 @@ func cleanDailyFiles(storagePath, subDir string, keepDay int) {
}
func CleanRecycleDiskfiles(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
for _, d := range options.HostOptions.LocalImagePath {
cleanDailyFiles(d, _RECYCLE_BIN_, options.HostOptions.RecycleDiskfileKeepDays)
cleanDailyFiles(d, _IMGSAVE_BACKUPS_, options.HostOptions.RecycleDiskfileKeepDays)
if storageManager == nil {
return
}
for _, storage := range storageManager.Storages {
storage.CleanRecycleDiskfiles(ctx)
}
}
func CleanImageCachefiles(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
if storageManager == nil {
return
}
storageManager.LocalStorageImagecacheManager.CleanImageCachefiles(ctx)
for _, imageCacheMan := range storageManager.LVMStorageImagecacheManagers {
imageCacheMan.CleanImageCachefiles(ctx)
}
for _, imageCacheMan := range storageManager.SharedLVMStorageImagecacheManagers {
imageCacheMan.CleanImageCachefiles(ctx)
}
for _, imageCacheMan := range storageManager.RbdStorageImagecacheManagers {
imageCacheMan.CleanImageCachefiles(ctx)
}
for _, imageCacheMan := range storageManager.SharedFileStorageImagecacheManagers {
imageCacheMan.CleanImageCachefiles(ctx)
}
}
pkg/hostman/storageman/disk_lvm.go
+1 -1
View File
@@ -329,7 +329,7 @@ func (d *SLVMDisk) createFromTemplate(
defer imageCacheManager.ReleaseImage(ctx, imageId)
cacheImagePath := imageCache.GetPath()
lvSizeMb := lvmutils.GetQcow2LvSize(imageCache.GetDesc().Size)
lvSizeMb := lvmutils.GetQcow2LvSize(imageCache.GetDesc().SizeMb)
if err := lvmutils.LvCreate(d.Storage.GetPath(), d.Id, lvSizeMb*1024*1024); err != nil {
return nil, errors.Wrap(err, "CreateRaw")
}
pkg/hostman/storageman/disk_nvme.go
+1 -1
View File
@@ -38,7 +38,7 @@ func (d *SNVMEDisk) GetDiskDesc() jsonutils.JSONObject {
var desc = jsonutils.NewDict()
desc.Set("disk_id", jsonutils.NewString(d.Id))
desc.Set("disk_size", jsonutils.NewInt(int64(d.Storage.GetCapacity())))
desc.Set("disk_size", jsonutils.NewInt(int64(d.Storage.GetCapacityMb())))
desc.Set("format", jsonutils.NewString(string(qemuimgfmt.RAW)))
desc.Set("disk_path", jsonutils.NewString(d.Storage.GetPath()))
return desc
pkg/hostman/storageman/imagecache_clean.go
+111
View File
@@ -0,0 +1,111 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package storageman
import (
"context"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
computeapis "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
baseoptions "yunion.io/x/onecloud/pkg/mcclient/options"
)
func cleanImages(ctx context.Context, manager IImageCacheManger, images map[string]IImageCache) (int64, error) {
if !manager.IsLocal() {
return 0, nil
}
storageCachedImages := make(map[string]computeapis.StoragecachedimageDetails)
limit := 50
total := -1
for total < 0 || len(storageCachedImages) < total {
params := baseoptions.BaseListOptions{}
details := true
params.Details = &details
params.Limit = &limit
offset := len(storageCachedImages)
params.Offset = &offset
result, err := compute.Storagecachedimages.ListDescendent(hostutils.GetComputeSession(ctx), manager.GetId(), jsonutils.Marshal(params))
if err != nil {
return 0, errors.Wrap(err, "List Storage Cached Images")
}
total = result.Total
for i := range result.Data {
ci := computeapis.StoragecachedimageDetails{}
err := result.Data[i].Unmarshal(&ci)
if err != nil {
return 0, errors.Wrap(err, "Unmarshal")
}
storageCachedImages[ci.CachedimageId] = ci
}
}
deleteSizeMb := int64(0)
for imageId, image := range images {
if _, ok := storageCachedImages[imageId]; !ok {
atime := image.GetDesc().AccessAt
if !atime.IsZero() && time.Now().Sub(atime) > time.Duration(options.HostOptions.ImageCacheExpireDays*86400)*time.Second {
continue
}
log.Infof("cached image %s not found on region, to delete size %dMB ...", imageId, image.GetDesc().SizeMb)
// not found on region, clean directly
if options.HostOptions.ImageCacheCleanupDryRun {
continue
}
err := manager.RemoveImage(ctx, imageId)
if err != nil {
return deleteSizeMb, errors.Wrapf(err, "RemoveImage %s", imageId)
}
deleteSizeMb += image.GetDesc().SizeMb
}
}
log.Infof("to delete non-exist image caches %dMB", deleteSizeMb)
for imgId := range storageCachedImages {
if _, ok := images[imgId]; !ok {
log.Infof("cached image %s in database not exists locally, to delete remotely ...", imgId)
_, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx), manager.GetId(), imgId, nil)
if err != nil {
log.Errorf("Fail to delete host cached image %s at %s: %s", imgId, manager.GetId(), err)
}
continue
}
img := storageCachedImages[imgId]
if img.Reference == 0 && (img.Size == 0 || time.Now().Sub(img.UpdatedAt) > time.Duration(options.HostOptions.ImageCacheExpireDays*86400)*time.Second) {
if img.Size == 0 {
img.Size = images[imgId].GetDesc().SizeMb * 1024 * 1024
}
log.Infof("image reference zero, to delete %s(%s) size %dMB", img.Cachedimage, img.CachedimageId, img.Size/1024/1024)
if options.HostOptions.ImageCacheCleanupDryRun {
continue
}
err := manager.RemoveImage(ctx, imgId)
if err != nil {
return deleteSizeMb, errors.Wrapf(err, "RemoveImage %s", imgId)
}
deleteSizeMb += img.Size / 1024 / 1024
}
}
return deleteSizeMb, nil
}
pkg/hostman/storageman/imagecache_local.go
+42 -20
View File
@@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/fs"
"os"
"path"
"sync"
@@ -89,7 +90,25 @@ func (l *SLocalImageCache) Load() error {
desc = &remotefile.SImageDesc{}
)
if fileutils2.Exists(imgPath) {
if !fileutils2.Exists(infPath) {
needReload := false
if fileutils2.Exists(infPath) {
sdesc, err := fileutils2.FileGetContents(infPath)
if err != nil {
return errors.Wrapf(err, "fileutils2.FileGetContents(%s)", infPath)
}
err = json.Unmarshal([]byte(sdesc), desc)
if err != nil {
return errors.Wrapf(err, "jsonutils.Unmarshal(%s)", infPath)
}
fi := l.getFileInfo()
if fi != nil && fi.Size()/1024/1024 != desc.SizeMb {
// fix file size
needReload = true
}
} else {
needReload = true
}
if needReload {
img, err := qemuimg.NewQemuImage(imgPath)
if err != nil {
return errors.Wrapf(err, "NewQemuImage(%s)", imgPath)
@@ -101,26 +120,27 @@ func (l *SLocalImageCache) Load() error {
if err != nil {
return errors.Wrapf(err, "fileutils2.MD5(%s)", imgPath)
}
desc = &remotefile.SImageDesc{
Format: string(img.Format),
Id: l.imageId,
Chksum: chksum,
Path: imgPath,
Size: l.GetSize(),
}
fi := l.getFileInfo()
if fi != nil {
desc.SizeMb = fi.Size() / 1024 / 1024
if fi.Sys() != nil {
atime := fi.Sys().(*syscall.Stat_t).Atim
desc.AccessAt = time.Unix(atime.Sec, atime.Nsec)
}
}
err = fileutils2.FilePutContents(infPath, jsonutils.Marshal(desc).PrettyString(), false)
if err != nil {
return errors.Wrapf(err, "fileutils2.FilePutContents(%s)", infPath)
}
} else {
sdesc, err := fileutils2.FileGetContents(infPath)
if err != nil {
return errors.Wrapf(err, "fileutils2.FileGetContents(%s)", infPath)
}
err = json.Unmarshal([]byte(sdesc), desc)
if err != nil {
return errors.Wrapf(err, "jsonutils.Unmarshal(%s)", infPath)
}
}
if len(desc.Chksum) > 0 && len(desc.Id) > 0 && desc.Id == l.imageId {
l.Desc = desc
@@ -211,8 +231,10 @@ func (l *SLocalImageCache) fetch(ctx context.Context, input api.CacheImageInput,
if err != nil {
return errors.Wrapf(err, "remoteFile.GetInfo")
}
l.Size = l.GetSize() / 1024 / 1024
fi := l.getFileInfo()
if fi != nil {
l.Size = fi.Size() / 1024 / 1024
}
l.Desc.Id = l.imageId
l.lastCheckTime = time.Now()
l.consumerCount++
@@ -252,17 +274,17 @@ func (l *SLocalImageCache) fetch(ctx context.Context, input api.CacheImageInput,
func (l *SLocalImageCache) Remove(ctx context.Context) error {
if fileutils2.Exists(l.GetPath()) {
if err := syscall.Unlink(l.GetPath()); err != nil {
return err
return errors.Wrap(err, l.GetPath())
}
}
if fileutils2.Exists(l.GetInfPath()) {
if err := syscall.Unlink(l.GetInfPath()); err != nil {
return err
return errors.Wrap(err, l.GetInfPath())
}
}
if fileutils2.Exists(l.GetTmpPath()) {
if err := syscall.Unlink(l.GetTmpPath()); err != nil {
return err
return errors.Wrap(err, l.GetTmpPath())
}
}
@@ -270,7 +292,7 @@ func (l *SLocalImageCache) Remove(ctx context.Context) error {
_, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx),
l.Manager.GetId(), l.imageId, nil)
if err != nil {
log.Errorf("Fail to delete host cached image: %s", err)
log.Errorf("Fail to delete host cached image %s at %s: %s", l.imageId, l.Manager.GetId(), err)
}
}()
@@ -289,11 +311,11 @@ func (l *SLocalImageCache) GetInfPath() string {
return l.GetPath() + _INF_SUFFIX_
}
func (l *SLocalImageCache) GetSize() int64 {
func (l *SLocalImageCache) getFileInfo() fs.FileInfo {
if fi, err := os.Stat(l.GetPath()); err != nil {
log.Errorln(err)
return 0
return nil
} else {
return fi.Size()
return fi
}
}
pkg/hostman/storageman/imagecache_lvm.go
+2 -2
View File
@@ -66,8 +66,8 @@ func (c *SLVMImageCache) GetDesc() *remotefile.SImageDesc {
}
return &remotefile.SImageDesc{
Size: sizeMb,
Name: c.GetName(),
SizeMb: sizeMb,
Name: c.GetName(),
}
}
pkg/hostman/storageman/imagecache_rbd.go
+4 -3
View File
@@ -134,8 +134,8 @@ func (r *SRbdImageCache) GetDesc() *remotefile.SImageDesc {
imageCacheManger := r.Manager.(*SRbdImageCacheManager)
desc := &remotefile.SImageDesc{
Size: -1,
Name: r.imageName,
SizeMb: -1,
Name: r.imageName,
}
cli, err := imageCacheManger.getCephClient()
@@ -153,7 +153,8 @@ func (r *SRbdImageCache) GetDesc() *remotefile.SImageDesc {
log.Errorf("GetInfo fail %s", err)
return desc
}
desc.Size = info.SizeByte / 1024 / 1024
desc.SizeMb = info.SizeByte / 1024 / 1024
desc.AccessAt = info.AccessTimestamp
return desc
}
pkg/hostman/storageman/imagecachemanager_base.go
+8 -1
View File
@@ -16,6 +16,7 @@ package storageman
import (
"context"
"sync"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
@@ -50,20 +51,26 @@ type IImageCacheManger interface {
SetStoragecacheId(string)
Lvmlockd() bool
IsLocal() bool
// for diskhandler
PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error)
DeleteImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error)
RemoveImage(ctx context.Context, imageId string) error
AcquireImage(ctx context.Context, input api.CacheImageInput, callback func(progress, progressMbps float64, totalSizeMb int64)) (IImageCache, error)
ReleaseImage(ctx context.Context, imageId string)
LoadImageCache(imageId string)
CleanImageCachefiles(ctx context.Context)
}
type SBaseImageCacheManager struct {
storageManager IStorageManager
storagecacaheId string
cachePath string
cachedImages map[string]IImageCache
cachedImages *sync.Map // map[string]IImageCache
}
func (c *SBaseImageCacheManager) GetPath() string {
pkg/hostman/storageman/imagecachemanager_local.go
+65 -21
View File
@@ -16,8 +16,8 @@ package storageman
import (
"context"
"io/ioutil"
"os"
"sync"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
@@ -27,6 +27,7 @@ import (
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
@@ -37,17 +38,20 @@ type SLocalImageCacheManager struct {
// limit int
// isTemplate bool
lock lockman.ILockManager
storage IStorage
}
func NewLocalImageCacheManager(manager IStorageManager, cachePath string, storagecacheId string) *SLocalImageCacheManager {
func NewLocalImageCacheManager(manager IStorageManager, cachePath string, storagecacheId string, storage IStorage) *SLocalImageCacheManager {
imageCacheManager := new(SLocalImageCacheManager)
imageCacheManager.lock = lockman.NewInMemoryLockManager()
imageCacheManager.storageManager = manager
imageCacheManager.storagecacaheId = storagecacheId
imageCacheManager.cachePath = cachePath
imageCacheManager.storage = storage
// imageCacheManager.limit = limit
// imageCacheManager.isTemplate = isTemplete
imageCacheManager.cachedImages = make(map[string]IImageCache, 0)
imageCacheManager.cachedImages = &sync.Map{} // make(map[string]IImageCache, 0)
if !fileutils2.Exists(cachePath) {
procutils.NewCommand("mkdir", "-p", cachePath).Run()
}
@@ -55,13 +59,20 @@ func NewLocalImageCacheManager(manager IStorageManager, cachePath string, storag
return imageCacheManager
}
func (c *SLocalImageCacheManager) IsLocal() bool {
if c.storage != nil {
return false
}
return true
}
func (c *SLocalImageCacheManager) loadCache(ctx context.Context) {
if len(c.cachePath) == 0 {
return
}
c.lock.LockRawObject(ctx, "LOCAL", "image-cache")
defer c.lock.ReleaseRawObject(ctx, "LOCAL", "image-cache")
files, _ := ioutil.ReadDir(c.cachePath)
files, _ := os.ReadDir(c.cachePath)
for _, f := range files {
if regutils.MatchUUIDExact(f.Name()) {
c.LoadImageCache(f.Name())
@@ -72,7 +83,7 @@ func (c *SLocalImageCacheManager) loadCache(ctx context.Context) {
func (c *SLocalImageCacheManager) LoadImageCache(imageId string) {
imageCache := NewLocalImageCache(imageId, c)
if imageCache.Load() == nil {
c.cachedImages[imageId] = imageCache
c.cachedImages.Store(imageId, imageCache)
}
}
@@ -80,18 +91,19 @@ func (c *SLocalImageCacheManager) AcquireImage(ctx context.Context, input api.Ca
c.lock.LockRawObject(ctx, "image-cache", input.ImageId)
defer c.lock.ReleaseRawObject(ctx, "image-cache", input.ImageId)
img, ok := c.cachedImages[input.ImageId]
imgObj, ok := c.cachedImages.Load(input.ImageId)
if !ok {
img = NewLocalImageCache(input.ImageId, c)
c.cachedImages[input.ImageId] = img
imgObj = NewLocalImageCache(input.ImageId, c)
c.cachedImages.Store(input.ImageId, imgObj)
}
if callback == nil && len(input.ServerId) > 0 {
callback = func(progress, progressMbps float64, totalSizeMb int64) {
if len(input.ServerId) > 0 {
hostutils.UpdateServerProgress(context.Background(), input.ServerId, progress, progressMbps)
hostutils.UpdateServerProgress(ctx, input.ServerId, progress, progressMbps)
}
}
}
img := imgObj.(IImageCache)
return img, img.Acquire(ctx, input, callback)
}
@@ -99,8 +111,8 @@ func (c *SLocalImageCacheManager) ReleaseImage(ctx context.Context, imageId stri
c.lock.LockRawObject(ctx, "image-cache", imageId)
defer c.lock.ReleaseRawObject(ctx, "image-cache", imageId)
if img, ok := c.cachedImages[imageId]; ok {
img.Release()
if img, ok := c.cachedImages.Load(imageId); ok {
img.(IImageCache).Release()
}
}
@@ -111,16 +123,16 @@ func (c *SLocalImageCacheManager) DeleteImageCache(ctx context.Context, data int
}
imageId, _ := body.GetString("image_id")
return nil, c.removeImage(ctx, imageId)
return nil, c.RemoveImage(ctx, imageId)
}
func (c *SLocalImageCacheManager) removeImage(ctx context.Context, imageId string) error {
func (c *SLocalImageCacheManager) RemoveImage(ctx context.Context, imageId string) error {
c.lock.LockRawObject(ctx, "image-cache", imageId)
defer c.lock.ReleaseRawObject(ctx, "image-cache", imageId)
if img, ok := c.cachedImages[imageId]; ok {
delete(c.cachedImages, imageId)
return img.Remove(ctx)
if img, ok := c.cachedImages.Load(imageId); ok {
c.cachedImages.Delete(imageId)
return img.(IImageCache).Remove(ctx)
}
return nil
}
@@ -156,14 +168,11 @@ func (c *SLocalImageCacheManager) PrefetchImageCache(ctx context.Context, data i
ret.ImageId = input.ImageId
ret.Path = imgCache.GetPath()
var (
size int64
)
if desc := imgCache.GetDesc(); desc != nil {
ret.Name = desc.Name
ret.Size = desc.Size
ret.Size = desc.SizeMb * 1024 * 1024 // ??? convert back to bytes?
}
if size == 0 {
if ret.Size == 0 {
fi, err := os.Stat(imgCache.GetPath())
if err != nil {
log.Errorf("os.Stat(%s) error: %v", imgCache.GetPath(), err)
@@ -177,3 +186,38 @@ func (c *SLocalImageCacheManager) PrefetchImageCache(ctx context.Context, data i
return jsonutils.Marshal(ret), nil
}
func (c *SLocalImageCacheManager) getTotalSize(ctx context.Context) (int64, map[string]IImageCache) {
total := int64(0)
images := make(map[string]IImageCache)
c.cachedImages.Range(func(imgId, imgObj any) bool {
img := imgObj.(IImageCache)
total += img.GetDesc().SizeMb
images[imgId.(string)] = img
return true
})
return total, images
}
func (c *SLocalImageCacheManager) CleanImageCachefiles(ctx context.Context) {
totalSize, images := c.getTotalSize(ctx)
storageSize := 0
if c.storage != nil {
// shared file storage
storageSize = c.storage.GetCapacityMb()
} else {
storageSize = c.storageManager.(*SStorageManager).GetTotalLocalCapacity()
}
ratio := float64(totalSize) / float64(storageSize)
log.Infof("SLocalImageCacheManager %s total size %dMB storage %dMB ratio %f expect ratio %d", c.cachePath, totalSize, storageSize, ratio, options.HostOptions.ImageCacheCleanupPercentage)
if int(ratio*100) < options.HostOptions.ImageCacheCleanupPercentage {
return
}
deletedMb, err := cleanImages(ctx, c, images)
if err != nil {
log.Errorf("SLocalImageCacheManager clean image %s fail %s", c.cachePath, err)
} else {
log.Infof("SLocalImageCacheManager %s cleanup %dMB", c.cachePath, deletedMb)
}
}
pkg/hostman/storageman/imagecachemanager_lvm.go
+52 -14
View File
@@ -17,6 +17,7 @@ package storageman
import (
"context"
"strings"
"sync"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
@@ -26,6 +27,7 @@ import (
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils"
"yunion.io/x/onecloud/pkg/httperrors"
)
@@ -35,23 +37,30 @@ const IMAGECACHE_PREFIX = "imagecache_"
type SLVMImageCacheManager struct {
SBaseImageCacheManager
storage IStorage
lvmlockd bool
lock lockman.ILockManager
}
func NewLVMImageCacheManager(manager IStorageManager, cachePath, storagecacheId string, lvmlockd bool) *SLVMImageCacheManager {
func NewLVMImageCacheManager(manager IStorageManager, cachePath, storagecacheId string, storage IStorage, lvmlockd bool) *SLVMImageCacheManager {
imageCacheManager := new(SLVMImageCacheManager)
imageCacheManager.lock = lockman.NewInMemoryLockManager()
imageCacheManager.storageManager = manager
imageCacheManager.storagecacaheId = storagecacheId
imageCacheManager.cachePath = cachePath
imageCacheManager.cachedImages = make(map[string]IImageCache, 0)
imageCacheManager.cachedImages = &sync.Map{} // make(map[string]IImageCache, 0)
imageCacheManager.storage = storage
imageCacheManager.lvmlockd = lvmlockd
imageCacheManager.loadCache(context.Background())
return imageCacheManager
}
func (c *SLVMImageCacheManager) IsLocal() bool {
return c.storage.IsLocal()
}
func (c *SLVMImageCacheManager) Lvmlockd() bool {
return c.lvmlockd
}
@@ -83,7 +92,7 @@ func (c *SLVMImageCacheManager) loadCache(ctx context.Context) {
func (c *SLVMImageCacheManager) LoadImageCache(imageId string) {
imageCache := NewLVMImageCache(imageId, c)
if err := imageCache.Load(); err == nil {
c.cachedImages[imageId] = imageCache
c.cachedImages.Store(imageId, imageCache)
} else {
log.Errorf("failed load cache %s %s", c.GetPath(), err)
}
@@ -96,10 +105,10 @@ func (c *SLVMImageCacheManager) AcquireImage(
c.lock.LockRawObject(ctx, "image-cache", input.ImageId)
defer c.lock.ReleaseRawObject(ctx, "image-cache", input.ImageId)
img, ok := c.cachedImages[input.ImageId]
imgObj, ok := c.cachedImages.Load(input.ImageId)
if !ok {
img = NewLVMImageCache(input.ImageId, c)
c.cachedImages[input.ImageId] = img
imgObj = NewLVMImageCache(input.ImageId, c)
c.cachedImages.Store(input.ImageId, imgObj)
}
if callback == nil && len(input.ServerId) > 0 {
callback = func(progress, progressMbps float64, totalSizeMb int64) {
@@ -108,6 +117,7 @@ func (c *SLVMImageCacheManager) AcquireImage(
}
}
}
img := imgObj.(IImageCache)
return img, img.Acquire(ctx, input, callback)
}
@@ -143,7 +153,7 @@ func (c *SLVMImageCacheManager) PrefetchImageCache(ctx context.Context, data int
if desc := cache.GetDesc(); desc != nil {
ret.Name = desc.Name
ret.Size = desc.Size
ret.Size = desc.SizeMb
}
return jsonutils.Marshal(ret), nil
}
@@ -155,16 +165,16 @@ func (c *SLVMImageCacheManager) DeleteImageCache(ctx context.Context, data inter
}
imageId, _ := body.GetString("image_id")
return nil, c.removeImage(ctx, imageId)
return nil, c.RemoveImage(ctx, imageId)
}
func (c *SLVMImageCacheManager) removeImage(ctx context.Context, imageId string) error {
func (c *SLVMImageCacheManager) RemoveImage(ctx context.Context, imageId string) error {
lockman.LockRawObject(ctx, "image-cache", imageId)
defer lockman.ReleaseRawObject(ctx, "image-cache", imageId)
if img, ok := c.cachedImages[imageId]; ok {
delete(c.cachedImages, imageId)
return img.Remove(ctx)
if img, ok := c.cachedImages.Load(imageId); ok {
c.cachedImages.Delete(imageId)
return img.(IImageCache).Remove(ctx)
}
return nil
}
@@ -172,7 +182,35 @@ func (c *SLVMImageCacheManager) removeImage(ctx context.Context, imageId string)
func (c *SLVMImageCacheManager) ReleaseImage(ctx context.Context, imageId string) {
lockman.LockRawObject(ctx, "image-cache", imageId)
defer lockman.ReleaseRawObject(ctx, "image-cache", imageId)
if img, ok := c.cachedImages[imageId]; ok {
img.Release()
if img, ok := c.cachedImages.Load(imageId); ok {
img.(IImageCache).Release()
}
}
func (c *SLVMImageCacheManager) getTotalSize(ctx context.Context) (int64, map[string]IImageCache) {
total := int64(0)
images := make(map[string]IImageCache, 0)
c.cachedImages.Range(func(imgId, imgObj any) bool {
img := imgObj.(IImageCache)
total += img.GetDesc().SizeMb
images[imgId.(string)] = img
return true
})
return total, images
}
func (c *SLVMImageCacheManager) CleanImageCachefiles(ctx context.Context) {
totalSize, images := c.getTotalSize(ctx)
ratio := float64(totalSize) / float64(c.storage.GetCapacityMb())
log.Infof("SLVMImageCacheManager %s total size %dMB storage capacity %dMB ratio %f expect ratio %d", c.cachePath, totalSize, c.storage.GetCapacityMb(), ratio, options.HostOptions.ImageCacheCleanupPercentage)
if int(ratio*100) < options.HostOptions.ImageCacheCleanupPercentage {
return
}
deletedMb, err := cleanImages(ctx, c, images)
if err != nil {
log.Errorf("SLVMImageCacheManager clean image %s fail %s", c.cachePath, err)
} else {
log.Infof("SLVMImageCacheManager %s cleanup %dMB", c.cachePath, deletedMb)
}
}
pkg/hostman/storageman/imagecachemanager_rbd.go
+48 -14
View File
@@ -17,6 +17,7 @@ package storageman
import (
"context"
"strings"
"sync"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
@@ -25,6 +26,7 @@ import (
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/hostman/hostutils"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/util/cephutils"
)
@@ -50,7 +52,7 @@ func NewRbdImageCacheManager(manager IStorageManager, cachePath string, storage
} else {
imageCacheManager.Pool, imageCacheManager.Prefix = cachePath, "image_cache_"
}
imageCacheManager.cachedImages = make(map[string]IImageCache, 0)
imageCacheManager.cachedImages = &sync.Map{} // make(map[string]IImageCache, 0)
imageCacheManager.loadCache(context.Background())
return imageCacheManager
}
@@ -104,10 +106,14 @@ func (c *SRbdImageCacheManager) loadCache(ctx context.Context) {
func (c *SRbdImageCacheManager) LoadImageCache(imageId string) {
imageCache := NewRbdImageCache(imageId, c)
if imageCache.Load() == nil {
c.cachedImages[imageId] = imageCache
c.cachedImages.Store(imageId, imageCache)
}
}
func (c *SRbdImageCacheManager) IsLocal() bool {
return false
}
func (c *SRbdImageCacheManager) GetPath() string {
return c.Pool
}
@@ -142,7 +148,7 @@ func (c *SRbdImageCacheManager) PrefetchImageCache(ctx context.Context, data int
if desc := cache.GetDesc(); desc != nil {
ret.Name = desc.Name
ret.Size = desc.Size
ret.Size = desc.SizeMb
}
return jsonutils.Marshal(ret), nil
}
@@ -154,16 +160,16 @@ func (c *SRbdImageCacheManager) DeleteImageCache(ctx context.Context, data inter
}
imageId, _ := body.GetString("image_id")
return nil, c.removeImage(ctx, imageId)
return nil, c.RemoveImage(ctx, imageId)
}
func (c *SRbdImageCacheManager) removeImage(ctx context.Context, imageId string) error {
func (c *SRbdImageCacheManager) RemoveImage(ctx context.Context, imageId string) error {
lockman.LockRawObject(ctx, "image-cache", imageId)
defer lockman.ReleaseRawObject(ctx, "image-cache", imageId)
if img, ok := c.cachedImages[imageId]; ok {
delete(c.cachedImages, imageId)
return img.Remove(ctx)
if img, ok := c.cachedImages.Load(imageId); ok {
c.cachedImages.Delete(imageId)
return img.(IImageCache).Remove(ctx)
}
return nil
}
@@ -172,19 +178,47 @@ func (c *SRbdImageCacheManager) AcquireImage(ctx context.Context, input api.Cach
lockman.LockRawObject(ctx, "image-cache", input.ImageId)
defer lockman.ReleaseRawObject(ctx, "image-cache", input.ImageId)
img, ok := c.cachedImages[input.ImageId]
imgObj, ok := c.cachedImages.Load(input.ImageId)
if !ok {
img = NewRbdImageCache(input.ImageId, c)
c.cachedImages[input.ImageId] = img
imgObj = NewRbdImageCache(input.ImageId, c)
c.cachedImages.Store(input.ImageId, imgObj)
}
img := imgObj.(IImageCache)
return img, img.Acquire(ctx, input, callback)
}
func (c *SRbdImageCacheManager) ReleaseImage(ctx context.Context, imageId string) {
lockman.LockRawObject(ctx, "image-cache", imageId)
defer lockman.ReleaseRawObject(ctx, "image-cache", imageId)
if img, ok := c.cachedImages[imageId]; ok {
img.Release()
if img, ok := c.cachedImages.Load(imageId); ok {
img.(IImageCache).Release()
}
}
func (c *SRbdImageCacheManager) getTotalSize(ctx context.Context) (int64, map[string]IImageCache) {
total := int64(0)
images := make(map[string]IImageCache, 0)
c.cachedImages.Range(func(imgId, imgObj any) bool {
img := imgObj.(IImageCache)
total += img.GetDesc().SizeMb
images[imgId.(string)] = img
return true
})
return total, images
}
func (c *SRbdImageCacheManager) CleanImageCachefiles(ctx context.Context) {
totalSize, images := c.getTotalSize(ctx)
ratio := float64(totalSize) / float64(c.storage.GetCapacityMb())
log.Infof("SRbdImageCacheManager %s total size %dMB storage capacity %dMB ratio %f expect ration %d", c.cachePath, totalSize, c.storage.GetCapacityMb(), ratio*100, options.HostOptions.ImageCacheCleanupPercentage)
if int(ratio*100) < options.HostOptions.ImageCacheCleanupPercentage {
return
}
deletedMb, err := cleanImages(ctx, c, images)
if err != nil {
log.Errorf("SRbdImageCacheManager clean image %s fail %s", c.cachePath, err)
} else {
log.Infof("SLocalImageCacheManager %s cleanup %dMB", c.cachePath, deletedMb)
}
}
pkg/hostman/storageman/remotefile/remotefile.go
+15 -6
View File
@@ -42,7 +42,9 @@ type SImageDesc struct {
Id string `json:"id"`
Chksum string `json:"chksum"`
Path string `json:"path"`
Size int64 `json:"size"`
SizeMb int64 `json:"size"`
AccessAt time.Time `json:"access_at"`
}
type SRemoteFile struct {
@@ -115,12 +117,19 @@ func (r *SRemoteFile) GetInfo() (*SImageDesc, error) {
return nil, errors.Wrapf(err, "os.Stat(%s)", r.localPath)
}
var atime time.Time
if fi.Sys() != nil {
atm := fi.Sys().(*syscall.Stat_t).Atim
atime = time.Unix(atm.Sec, atm.Nsec)
}
return &SImageDesc{
Name: r.name,
Format: r.format,
Chksum: r.chksum,
Path: r.localPath,
Size: fi.Size(),
Name: r.name,
Format: r.format,
Chksum: r.chksum,
Path: r.localPath,
SizeMb: fi.Size() / 1024 / 1024,
AccessAt: atime,
}, nil
}
pkg/hostman/storageman/storage_base.go
+5 -3
View File
@@ -109,7 +109,7 @@ type IStorage interface {
StorageBackupRecovery(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)
GetFreeSizeMb() int
GetCapacity() int
GetCapacityMb() int
// Find owner disks first, if not found, call create disk
GetDiskById(diskId string) (IDisk, error)
@@ -143,6 +143,8 @@ type IStorage interface {
Accessible() error
Detach() error
CleanRecycleDiskfiles(ctx context.Context)
}
type SBaseStorage struct {
@@ -223,7 +225,7 @@ func (s *SBaseStorage) GetZoneId() string {
return s.Manager.GetZoneId()
}
func (s *SBaseStorage) GetCapacity() int {
func (s *SBaseStorage) GetCapacityMb() int {
return s.GetAvailSizeMb()
}
@@ -280,7 +282,7 @@ func (s *SBaseStorage) SyncStorageSize() (api.SHostStorageStat, error) {
stat := api.SHostStorageStat{
StorageId: s.StorageId,
}
stat.CapacityMb = int64(s.GetCapacity())
stat.CapacityMb = int64(s.GetCapacityMb())
stat.ActualCapacityUsedMb = int64(s.GetUsedSizeMb())
return stat, nil
}
pkg/hostman/storageman/storage_local.go
+5
View File
@@ -925,3 +925,8 @@ func (s *SLocalStorage) CloneDiskFromStorage(
TargetFormat: qemuimgfmt.QCOW2.String(),
}, nil
}
func (s *SLocalStorage) CleanRecycleDiskfiles(ctx context.Context) {
cleanDailyFiles(s.Path, _RECYCLE_BIN_, options.HostOptions.RecycleDiskfileKeepDays)
cleanDailyFiles(s.Path, _IMGSAVE_BACKUPS_, options.HostOptions.RecycleDiskfileKeepDays)
}
pkg/hostman/storageman/storage_lvm.go
+5 -1
View File
@@ -161,7 +161,7 @@ func (s *SLVMStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
if err == nil {
log.Errorf("storage created %s", res)
storageCacheId, _ := res.GetString("storagecache_id")
storageManager.InitLVMStorageImageCache(storageCacheId, s.GetPath())
storageManager.InitLVMStorageImageCache(storageCacheId, s.GetPath(), s)
s.SetStoragecacheId(storageCacheId)
}
}
@@ -532,3 +532,7 @@ func (s *SLVMStorage) CloneDiskFromStorage(
TargetFormat: qemuimgfmt.QCOW2.String(),
}, nil
}
func (s *SLVMStorage) CleanRecycleDiskfiles(ctx context.Context) {
log.Infof("SLVMStorage CleanRecycleDiskfiles do nothing!")
}
pkg/hostman/storageman/storage_nvme.go
+5 -1
View File
@@ -38,7 +38,7 @@ func (s *SNVMEStorage) GetAvailSizeMb() int {
return s.sizeMB
}
func (s *SNVMEStorage) GetCapacity() int {
func (s *SNVMEStorage) GetCapacityMb() int {
return s.GetAvailSizeMb()
}
@@ -194,3 +194,7 @@ func (s *SNVMEStorage) GetComposedName() string {
p = strings.ReplaceAll(s.Path, ":", "_")
return fmt.Sprintf("host_%s_%s_storage_%s", s.Manager.host.GetMasterIp(), s.StorageType(), p)
}
func (s *SNVMEStorage) CleanRecycleDiskfiles(ctx context.Context) {
log.Infof("SNVMEStorage CleanRecycleDiskfiles do nothing!")
}
pkg/hostman/storageman/storage_rbd.go
+27 -9
View File
@@ -17,7 +17,6 @@ package storageman
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
@@ -403,16 +402,31 @@ func (s *SRbdStorage) SyncStorageSize() (api.SHostStorageStat, error) {
return stat, nil
}
func (s *SRbdStorage) GetCapacityMb() int {
capa, err := s.getRbdCapacity()
if err != nil {
return -1
}
return int(capa.CapacitySizeKb) / 1024
}
func (s *SRbdStorage) getRbdCapacity() (*cephutils.SCapacity, error) {
client, err := s.getClient()
if err != nil {
return nil, errors.Wrap(err, "getClient")
}
defer client.Close()
capacity, err := client.GetCapacity()
if err != nil {
return nil, errors.Wrap(err, "GetCapacity")
}
return capacity, nil
}
func (s *SRbdStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
content := map[string]interface{}{}
if len(s.StorageId) > 0 {
client, err := s.getClient()
if err != nil {
reason := jsonutils.Marshal(map[string]string{"reason": errors.Wrapf(err, "GetClient").Error()})
return modules.Storages.PerformAction(hostutils.GetComputeSession(context.Background()), s.StorageId, api.STORAGE_OFFLINE, reason)
}
defer client.Close()
capacity, err := client.GetCapacity()
capacity, err := s.getRbdCapacity()
if err != nil {
reason := jsonutils.Marshal(map[string]string{"reason": errors.Wrapf(err, "GetCapacity").Error()})
return modules.Storages.PerformAction(hostutils.GetComputeSession(context.Background()), s.StorageId, api.STORAGE_OFFLINE, reason)
@@ -518,7 +532,7 @@ func (s *SRbdStorage) saveToGlance(ctx context.Context, imageId, imagePath strin
return err
}
tmpFileDir, err := ioutil.TempDir(options.HostOptions.TempPath, "ceph_save_images")
tmpFileDir, err := os.MkdirTemp(options.HostOptions.TempPath, "ceph_save_images")
if err != nil {
log.Errorf("fail to obtain tempFile for ceph save glance image: %s", err)
return errors.Wrap(err, "ioutil.TempDir")
@@ -684,3 +698,7 @@ func (s *SRbdStorage) SetStorageInfo(storageId, storageName string, conf jsonuti
}
return nil
}
func (s *SRbdStorage) CleanRecycleDiskfiles(ctx context.Context) {
log.Infof("SRbdStorage CleanRecycleDiskfiles do nothing!")
}
pkg/util/cephutils/ceph.go
+4 -3
View File
@@ -20,6 +20,7 @@ import (
"io/ioutil"
"os"
"strings"
"time"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/jsonutils"
@@ -320,9 +321,9 @@ type SImageInfo struct {
Features []string `json:"features"`
OpFeatures []interface{} `json:"op_features"`
Flags []interface{} `json:"flags"`
CreateTimestamp string `json:"create_timestamp"`
AccessTimestamp string `json:"access_timestamp"`
ModifyTimestamp string `json:"modify_timestamp"`
CreateTimestamp time.Time `json:"create_timestamp"`
AccessTimestamp time.Time `json:"access_timestamp"`
ModifyTimestamp time.Time `json:"modify_timestamp"`
}
func (img *SImage) options() []string {