fix: storagecachedimages details speedup (#20234)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

pkg/apis/compute/storagecache.go

@@ -100,6 +100,26 @@ type StoragecacheResourceInfo struct {
 
 	// 存储缓存名称
 	Storagecache string `json:"storagecache"`
+
+	// 关联存储名称
+	Storages []string `json:"storages"`
+
+	// 关联存储信息
+	StorageInfo []StorageInfo `json:"storage_info"`
+}
+
+type StorageInfo struct {
+	Id string `json:"id"`
+
+	Name string `json:"name"`
+
+	StorageType string `json:"storage_type"`
+
+	MediumType string `json:"medium_type"`
+
+	ZoneId string `json:"zone_id"`
+
+	Zone string `json:"zone"`
 }
 
 type StoragecacheFilterListInputBase struct {

pkg/apis/compute/storagecachedimage.go

@@ -15,8 +15,6 @@
 package compute
 
 import (
-	"yunion.io/x/jsonutils"
-
 	"yunion.io/x/onecloud/pkg/apis"
 )
 
@@ -31,15 +29,20 @@ type StoragecachedimageDetails struct {
 	Cachedimage string `json:"cachedimage"`
 
 	// 存储列表
-	Storages []string `json:"storages"`
+	// Storages []string `json:"storages"`
 	// 通过一致性哈希获取的一个宿主机详情
-	Host *jsonutils.JSONDict `json:"host"`
+	// Host *jsonutils.JSONDict `json:"host"`
+
 	// 镜像名称
 	Image string `json:"image"`
 	// 镜像大小
 	Size int64 `json:"size"`
 	// 引用次数
 	Reference int `json:"reference"`
+	// Disk引用次数
+	DiskReference int `json:"disk_reference"`
+	// Cdrom引用次数
+	CdromReference int `json:"cdrom_reference"`
 }
 
 type StoragecachedimageListInput struct {

pkg/cloudcommon/db/db_joint_dispatcher.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/printutils"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
@@ -74,10 +75,12 @@ func (dispatcher *DBJointModelDispatcher) ListMasterDescendent(ctx context.Conte
 	ctx = manager.PrepareQueryContext(ctx, userCred, query)
 
 	model, err := fetchItem(manager.GetMasterManager(), ctx, userCred, idStr, query)
-	if err == sql.ErrNoRows {
-		return nil, httperrors.NewResourceNotFoundError2(manager.GetMasterManager().Keyword(), idStr)
-	} else if err != nil {
-		return nil, err
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2(manager.GetMasterManager().Keyword(), idStr)
+		} else {
+			return nil, err
+		}
 	}
 	queryDict.Add(jsonutils.NewString(model.GetId()), fmt.Sprintf("%s_id", manager.GetMasterManager().Keyword()))
 	queryDict.Add(jsonutils.NewString(model.GetId()), manager.GetMasterFieldName())
@@ -104,10 +107,12 @@ func (dispatcher *DBJointModelDispatcher) ListSlaveDescendent(ctx context.Contex
 	ctx = manager.PrepareQueryContext(ctx, userCred, query)
 
 	model, err := fetchItem(manager.GetSlaveManager(), ctx, userCred, idStr, query)
-	if err == sql.ErrNoRows {
-		return nil, httperrors.NewResourceNotFoundError2(manager.GetSlaveManager().Keyword(), idStr)
-	} else if err != nil {
-		return nil, err
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2(manager.GetSlaveManager().Keyword(), idStr)
+		} else {
+			return nil, httperrors.NewGeneralError(err)
+		}
 	}
 	queryDict.Add(jsonutils.NewString(model.GetId()), fmt.Sprintf("%s_id", manager.GetSlaveManager().Keyword()))
 	queryDict.Add(jsonutils.NewString(model.GetId()), manager.GetSlaveFieldName())
@@ -130,15 +135,27 @@ func _listJoint(manager IModelManager, ctx context.Context, userCred mcclient.To
 func fetchJointItem(manager IJointModelManager, ctx context.Context, userCred mcclient.TokenCredential, id1 string, id2 string, query jsonutils.JSONObject) (IStandaloneModel, IStandaloneModel, IJointModel, error) {
 	master, err := fetchItem(manager.GetMasterManager(), ctx, userCred, id1, query)
 	if err != nil {
-		return nil, nil, nil, httperrors.NewGeneralError(err)
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, nil, nil, httperrors.NewResourceNotFoundError2(manager.GetMasterManager().Keyword(), id1)
+		} else {
+			return nil, nil, nil, httperrors.NewGeneralError(err)
+		}
 	}
 	slave, err := fetchItem(manager.GetSlaveManager(), ctx, userCred, id2, query)
 	if err != nil {
-		return nil, nil, nil, httperrors.NewGeneralError(err)
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, nil, nil, httperrors.NewResourceNotFoundError2(manager.GetSlaveManager().Keyword(), id2)
+		} else {
+			return nil, nil, nil, httperrors.NewGeneralError(err)
+		}
 	}
 	item, err := FetchJointByIds(manager, master.GetId(), slave.GetId(), query)
 	if err != nil {
-		return nil, nil, nil, err
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, nil, nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), id1+"-"+id2)
+		} else {
+			return nil, nil, nil, httperrors.NewGeneralError(err)
+		}
 	}
 	return master.(IStandaloneModel), slave.(IStandaloneModel), item, nil
 }

pkg/cloudcommon/db/fetch.go

@@ -54,6 +54,7 @@ func FetchJointByIds(manager IJointModelManager, masterId, slaveId string, query
 	}
 	cond := sqlchemy.AND(sqlchemy.Equals(masterField, masterId), sqlchemy.Equals(slaveField, slaveId))
 	q = q.Filter(cond)
+	q.DebugQuery()
 	q = manager.FilterByParams(q, query)
 	count, err := q.CountWithError()
 	if err != nil {

pkg/compute/models/storagecachedimages.go

@@ -128,6 +128,9 @@ func (manager *SStoragecachedimageManager) FetchCustomizeColumns(
 ) []api.StoragecachedimageDetails {
 	rows := make([]api.StoragecachedimageDetails, len(objs))
 
+	storagecacheIds := make([]string, 0)
+	imageIds := make([]string, 0)
+
 	jointRows := manager.SJointResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	scRows := manager.SStoragecacheResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	for i := range rows {
@@ -135,7 +138,45 @@ func (manager *SStoragecachedimageManager) FetchCustomizeColumns(
 			JointResourceBaseDetails: jointRows[i],
 			StoragecacheResourceInfo: scRows[i],
 		}
-		rows[i] = objs[i].(*SStoragecachedimage).getExtraDetails(ctx, rows[i])
+		sci := objs[i].(*SStoragecachedimage)
+		storagecacheIds = append(storagecacheIds, sci.StoragecacheId)
+		imageIds = append(imageIds, sci.CachedimageId)
+	}
+
+	cachedImages := make(map[string]SCachedimage)
+	err := db.FetchModelObjectsByIds(CachedimageManager, "id", imageIds, &cachedImages)
+	if err != nil {
+		log.Errorf("db.FetchModelObjectsByIds fail %s", err)
+	}
+	cdromRefs, err := manager.fetchCdromReferenceCounts(storagecacheIds, imageIds)
+	if err != nil {
+		log.Errorf("manager.fetchCdromReferenceCounts fail %s", err)
+	}
+	diskRefs, err := manager.fetchDiskReferenceCounts(storagecacheIds, imageIds)
+	if err != nil {
+		log.Errorf("manager.fetchDiskReferenceCounts fail %s", err)
+	}
+
+	for i := range rows {
+		sci := objs[i].(*SStoragecachedimage)
+		if cachedImages != nil {
+			if cachedImage, ok := cachedImages[sci.CachedimageId]; ok {
+				rows[i].Cachedimage = cachedImage.Name
+				rows[i].Image = cachedImage.Name
+				rows[i].Size = cachedImage.Size
+			}
+		}
+		if cdromRefs != nil {
+			if refMap, ok := cdromRefs[sci.StoragecacheId]; ok {
+				rows[i].CdromReference = refMap[sci.CachedimageId]
+			}
+		}
+		if diskRefs != nil {
+			if refMap, ok := diskRefs[sci.StoragecacheId]; ok {
+				rows[i].DiskReference = refMap[sci.CachedimageId]
+				rows[i].Reference = rows[i].CdromReference + rows[i].DiskReference
+			}
+		}
 	}
 
 	return rows
@@ -149,7 +190,7 @@ func (self *SStoragecachedimage) GetCachedimage() *SCachedimage {
 	return nil
 }
 
-func (self *SStoragecachedimage) getExtraDetails(ctx context.Context, out api.StoragecachedimageDetails) api.StoragecachedimageDetails {
+/*func (self *SStoragecachedimage) getExtraDetails(ctx context.Context, out api.StoragecachedimageDetails) api.StoragecachedimageDetails {
 	storagecache := self.GetStoragecache()
 	if storagecache != nil {
 		// out.Storagecache = storagecache.Name
@@ -176,6 +217,54 @@ func (self *SStoragecachedimage) getExtraDetails(ctx context.Context, out api.St
 	}
 	out.Reference, _ = self.getReferenceCount()
 	return out
+}*/
+
+func (manager *SStoragecachedimageManager) fetchCdromReferenceCounts(storagecacheIds []string, imageIds []string) (map[string]map[string]int, error) {
+	q := GuestcdromManager.Query()
+	guests := GuestManager.Query().SubQuery()
+	hostStorages := HoststorageManager.Query().SubQuery()
+	storages := StorageManager.Query().SubQuery()
+
+	q = q.Join(guests, sqlchemy.Equals(q.Field("id"), guests.Field("id")))
+	q = q.Join(hostStorages, sqlchemy.Equals(guests.Field("host_id"), hostStorages.Field("host_id")))
+	q = q.Join(storages, sqlchemy.Equals(hostStorages.Field("storage_id"), storages.Field("id")))
+
+	q = q.GroupBy(q.Field("image_id"))
+	q = q.GroupBy(storages.Field("storagecache_id"))
+
+	q = q.Filter(sqlchemy.In(q.Field("image_id"), imageIds))
+	q = q.Filter(sqlchemy.In(storages.Field("storagecache_id"), storagecacheIds))
+
+	q = q.AppendField(sqlchemy.COUNT("ref_count"))
+	q = q.AppendField(q.Field("image_id"))
+	q = q.AppendField(storages.Field("storagecache_id"))
+
+	return manager.fetchRefCount(q)
+}
+
+func (maanger *SStoragecachedimageManager) fetchRefCount(q *sqlchemy.SQuery) (map[string]map[string]int, error) {
+	results := []struct {
+		RefCount       int    `json:"ref_count"`
+		ImageId        string `json:"image_id"`
+		StoragecacheId string `json:"storagecache_id"`
+	}{}
+
+	q.DebugQuery()
+
+	err := q.All(&results)
+	if err != nil {
+		return nil, errors.Wrap(err, "Query")
+	}
+
+	ret := make(map[string]map[string]int)
+	for _, r := range results {
+		if _, ok := ret[r.StoragecacheId]; !ok {
+			ret[r.StoragecacheId] = make(map[string]int)
+		}
+		ret[r.StoragecacheId][r.ImageId] = r.RefCount
+	}
+
+	return ret, nil
 }
 
 func (self *SStoragecachedimage) getCdromReferenceCount() (int, error) {
@@ -188,6 +277,27 @@ func (self *SStoragecachedimage) getCdromReferenceCount() (int, error) {
 	return q.CountWithError()
 }
 
+func (manager *SStoragecachedimageManager) fetchDiskReferenceCounts(storagecacheIds, imageIds []string) (map[string]map[string]int, error) {
+	disks := DiskManager.Query().SubQuery()
+	storages := StorageManager.Query().SubQuery()
+
+	q := disks.Query()
+
+	q = q.Join(storages, sqlchemy.Equals(disks.Field("storage_id"), storages.Field("id")))
+
+	q = q.GroupBy(disks.Field("template_id"))
+	q = q.GroupBy(storages.Field("storagecache_id"))
+
+	q = q.Filter(sqlchemy.In(disks.Field("template_id"), imageIds))
+	q = q.Filter(sqlchemy.In(storages.Field("storagecache_id"), storagecacheIds))
+
+	q = q.AppendField(sqlchemy.COUNT("ref_count"))
+	q = q.AppendField(disks.Field("template_id").Label("image_id"))
+	q = q.AppendField(storages.Field("storagecache_id"))
+
+	return manager.fetchRefCount(q)
+}
+
 func (self *SStoragecachedimage) getDiskReferenceCount() (int, error) {
 	guestdisks := GuestdiskManager.Query().SubQuery()
 	disks := DiskManager.Query().SubQuery()

pkg/compute/models/storagecacheresource.go

@@ -87,6 +87,27 @@ func (manager *SStoragecacheResourceBaseManager) FetchCustomizeColumns(
 		return nil
 	}
 
+	storageMap := make(map[string][]api.StorageInfo, 0)
+	{
+		q := StorageManager.Query("id", "name", "storage_type", "medium_type", "storagecache_id", "zone_id").In("storagecache_id", storagecacheIds)
+		zones := ZoneManager.Query().SubQuery()
+		q = q.Join(zones, sqlchemy.Equals(q.Field("zone_id"), zones.Field("id")))
+		q = q.AppendField(zones.Field("name").Label("zone"))
+
+		storages := make([]struct {
+			api.StorageInfo
+			StoragecacheId string `json:"storagecache_id"`
+		}, 0)
+		err := q.All(&storages)
+		if err != nil {
+			log.Errorf("Storage Info Query query fail %s", err)
+		} else {
+			for _, si := range storages {
+				storageMap[si.StoragecacheId] = append(storageMap[si.StoragecacheId], si.StorageInfo)
+			}
+		}
+	}
+
 	managerList := make([]interface{}, len(rows))
 
 	for i := range rows {
@@ -96,6 +117,12 @@ func (manager *SStoragecacheResourceBaseManager) FetchCustomizeColumns(
 			rows[i].Storagecache = storagecache.Name
 			rows[i].ManagerId = storagecache.ManagerId
 		}
+		if info, ok := storageMap[storagecacheIds[i]]; ok {
+			rows[i].StorageInfo = info
+			for i := range info {
+				rows[i].Storages = append(rows[i].Storages, info[i].Name)
+			}
+		}
 		managerList[i] = &SManagedResourceBase{rows[i].ManagerId}
 	}
 

pkg/hostman/storageman/imagecache_local.go

@@ -29,6 +29,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/httputils"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -295,7 +296,7 @@ func (l *SLocalImageCache) Remove(ctx context.Context) error {
 	go func() {
 		_, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx),
 			l.Manager.GetId(), l.imageId, nil)
-		if err != nil {
+		if err != nil && httputils.ErrorCode(err) != 404 {
 			log.Errorf("Fail to delete host cached image %s at %s: %s", l.imageId, l.Manager.GetId(), err)
 		}
 	}()

pkg/hostman/storageman/imagecache_lvm.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/httputils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
@@ -151,7 +152,7 @@ func (c *SLVMImageCache) Remove(ctx context.Context) error {
 	go func() {
 		_, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx),
 			c.Manager.GetId(), c.imageId, nil)
-		if err != nil {
+		if err != nil && httputils.ErrorCode(err) != 404 {
 			log.Errorf("Fail to delete host cached image: %s", err)
 		}
 	}()

pkg/hostman/storageman/imagecache_rbd.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/httputils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
@@ -123,7 +124,7 @@ func (r *SRbdImageCache) Remove(ctx context.Context) error {
 	go func() {
 		_, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx),
 			r.Manager.GetId(), r.imageId, nil)
-		if err != nil {
+		if err != nil && httputils.ErrorCode(err) != 404 {
 			log.Errorf("Fail to delete host cached image: %s", err)
 		}
 	}()