diff --git a/pkg/cloudcommon/db/fetch.go b/pkg/cloudcommon/db/fetch.go index 45417019ee..69a71eaa00 100644 --- a/pkg/cloudcommon/db/fetch.go +++ b/pkg/cloudcommon/db/fetch.go @@ -19,6 +19,8 @@ import ( "database/sql" "fmt" "reflect" + "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" @@ -602,3 +604,59 @@ func FetchDistinctField(modelManager IModelManager, field string) ([]string, err return q.Distinct() }) } + +func Purge(modelManager IModelManager, field string, ids []string, forceDelete bool) error { + if len(ids) == 0 { + return nil + } + + var splitByLen = func(data []string, splitLen int) [][]string { + var result [][]string + for i := 0; i < len(data); i += splitLen { + end := i + splitLen + if end > len(data) { + end = len(data) + } + result = append(result, data[i:end]) + } + return result + } + + var purge = func(ids []string) error { + vars := []interface{}{} + placeholders := make([]string, len(ids)) + for i := range placeholders { + placeholders[i] = "?" + vars = append(vars, ids[i]) + } + placeholder := strings.Join(placeholders, ",") + sql := fmt.Sprintf( + "delete from %s where %s in (%s)", + modelManager.TableSpec().Name(), field, placeholder, + ) + + if !forceDelete { + sql = fmt.Sprintf( + "update %s set deleted=1, deleted_at= ? where %s in (%s)", + modelManager.TableSpec().Name(), field, placeholder, + ) + vars = append([]interface{}{time.Now()}, vars...) + } + _, err := sqlchemy.GetDB().Exec( + sql, vars..., + ) + if err != nil { + return errors.Wrapf(err, strings.ReplaceAll(sql, "?", "%s"), vars...) + } + return nil + } + + idsArr := splitByLen(ids, 100) + for i := range idsArr { + err := purge(idsArr[i]) + if err != nil { + return errors.Wrapf(err, "purge") + } + } + return nil +} diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index bbb1a64296..17ff490435 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -945,6 +945,10 @@ func (manager *SCachedimageManager) AutoCleanImageCaches(ctx context.Context, us if err != nil { log.Errorf("cleanExternalImages error: %v", err) } + err = manager.cleanStoragecachedimages() + if err != nil { + log.Errorf("cleanStoragecachedimages error: %v", err) + } }() lastSync := time.Now().Add(time.Duration(-1*api.CACHED_IMAGE_REFERENCE_SESSION_EXPIRE_SECONDS) * time.Second) q := manager.Query() @@ -1020,55 +1024,32 @@ func (manager *SCachedimageManager) cleanExternalImages() error { return errors.Wrapf(err, "getExpireExternalImageIds") } - if len(ids) == 0 { - return nil - } - - var splitByLen = func(data []string, splitLen int) [][]string { - var result [][]string - for i := 0; i < len(data); i += splitLen { - end := i + splitLen - if end > len(data) { - end = len(data) - } - result = append(result, data[i:end]) - } - return result - } - - var purge = func(ids []string) error { - vars := []interface{}{} - placeholders := make([]string, len(ids)) - for i := range placeholders { - placeholders[i] = "?" - vars = append(vars, ids[i]) - } - placeholder := strings.Join(placeholders, ",") - sql := fmt.Sprintf( - "delete from %s where id in (%s)", - manager.TableSpec().Name(), placeholder, - ) - _, err = sqlchemy.GetDB().Exec( - sql, vars..., - ) - if err != nil { - return errors.Wrapf(err, strings.ReplaceAll(sql, "?", "%s"), vars...) - } - return nil - } - - idsArr := splitByLen(ids, 100) - for i := range idsArr { - err = purge(idsArr[i]) - if err != nil { - return errors.Wrapf(err, "purge") - } + err = db.Purge(manager, "id", ids, true) + if err != nil { + return errors.Wrapf(err, "purge") } log.Debugf("clean %d expired external images", len(ids)) return nil } +func (manager *SCachedimageManager) cleanStoragecachedimages() error { + ids, err := db.FetchField(StoragecachedimageManager, "row_id", func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + sq := manager.Query("id").Distinct().SubQuery() + return q.NotIn("cachedimage_id", sq) + }) + if err != nil { + return errors.Wrapf(err, "getExpireExternalImageIds") + } + err = db.Purge(StoragecachedimageManager, "row_id", ids, true) + if err != nil { + return errors.Wrapf(err, "purge") + } + + log.Debugf("clean %d invalid storagecachedimages", len(ids)) + return nil +} + func (image *SCachedimage) GetAllClassMetadata() (map[string]string, error) { meta, err := image.SSharableVirtualResourceBase.GetAllClassMetadata() if err != nil {