diff --git a/pkg/apis/compute/storagecache.go b/pkg/apis/compute/storagecache.go index f706e5a1d7..9700812f5d 100644 --- a/pkg/apis/compute/storagecache.go +++ b/pkg/apis/compute/storagecache.go @@ -100,6 +100,26 @@ type StoragecacheResourceInfo struct { // 存储缓存名称 Storagecache string `json:"storagecache"` + + // 关联存储名称 + Storages []string `json:"storages"` + + // 关联存储信息 + StorageInfo []StorageInfo `json:"storage_info"` +} + +type StorageInfo struct { + Id string `json:"id"` + + Name string `json:"name"` + + StorageType string `json:"storage_type"` + + MediumType string `json:"medium_type"` + + ZoneId string `json:"zone_id"` + + Zone string `json:"zone"` } type StoragecacheFilterListInputBase struct { diff --git a/pkg/apis/compute/storagecachedimage.go b/pkg/apis/compute/storagecachedimage.go index 41b0291b9f..bfde4f8709 100644 --- a/pkg/apis/compute/storagecachedimage.go +++ b/pkg/apis/compute/storagecachedimage.go @@ -15,8 +15,6 @@ package compute import ( - "yunion.io/x/jsonutils" - "yunion.io/x/onecloud/pkg/apis" ) @@ -31,15 +29,20 @@ type StoragecachedimageDetails struct { Cachedimage string `json:"cachedimage"` // 存储列表 - Storages []string `json:"storages"` + // Storages []string `json:"storages"` // 通过一致性哈希获取的一个宿主机详情 - Host *jsonutils.JSONDict `json:"host"` + // Host *jsonutils.JSONDict `json:"host"` + // 镜像名称 Image string `json:"image"` // 镜像大小 Size int64 `json:"size"` // 引用次数 Reference int `json:"reference"` + // Disk引用次数 + DiskReference int `json:"disk_reference"` + // Cdrom引用次数 + CdromReference int `json:"cdrom_reference"` } type StoragecachedimageListInput struct { diff --git a/pkg/cloudcommon/db/db_joint_dispatcher.go b/pkg/cloudcommon/db/db_joint_dispatcher.go index d3cd414ebe..5f501b34d0 100644 --- a/pkg/cloudcommon/db/db_joint_dispatcher.go +++ b/pkg/cloudcommon/db/db_joint_dispatcher.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/printutils" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" @@ -74,10 +75,12 @@ func (dispatcher *DBJointModelDispatcher) ListMasterDescendent(ctx context.Conte ctx = manager.PrepareQueryContext(ctx, userCred, query) model, err := fetchItem(manager.GetMasterManager(), ctx, userCred, idStr, query) - if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(manager.GetMasterManager().Keyword(), idStr) - } else if err != nil { - return nil, err + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(manager.GetMasterManager().Keyword(), idStr) + } else { + return nil, err + } } queryDict.Add(jsonutils.NewString(model.GetId()), fmt.Sprintf("%s_id", manager.GetMasterManager().Keyword())) queryDict.Add(jsonutils.NewString(model.GetId()), manager.GetMasterFieldName()) @@ -104,10 +107,12 @@ func (dispatcher *DBJointModelDispatcher) ListSlaveDescendent(ctx context.Contex ctx = manager.PrepareQueryContext(ctx, userCred, query) model, err := fetchItem(manager.GetSlaveManager(), ctx, userCred, idStr, query) - if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(manager.GetSlaveManager().Keyword(), idStr) - } else if err != nil { - return nil, err + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(manager.GetSlaveManager().Keyword(), idStr) + } else { + return nil, httperrors.NewGeneralError(err) + } } queryDict.Add(jsonutils.NewString(model.GetId()), fmt.Sprintf("%s_id", manager.GetSlaveManager().Keyword())) queryDict.Add(jsonutils.NewString(model.GetId()), manager.GetSlaveFieldName()) @@ -130,15 +135,27 @@ func _listJoint(manager IModelManager, ctx context.Context, userCred mcclient.To func fetchJointItem(manager IJointModelManager, ctx context.Context, userCred mcclient.TokenCredential, id1 string, id2 string, query jsonutils.JSONObject) (IStandaloneModel, IStandaloneModel, IJointModel, error) { master, err := fetchItem(manager.GetMasterManager(), ctx, userCred, id1, query) if err != nil { - return nil, nil, nil, httperrors.NewGeneralError(err) + if errors.Cause(err) == sql.ErrNoRows { + return nil, nil, nil, httperrors.NewResourceNotFoundError2(manager.GetMasterManager().Keyword(), id1) + } else { + return nil, nil, nil, httperrors.NewGeneralError(err) + } } slave, err := fetchItem(manager.GetSlaveManager(), ctx, userCred, id2, query) if err != nil { - return nil, nil, nil, httperrors.NewGeneralError(err) + if errors.Cause(err) == sql.ErrNoRows { + return nil, nil, nil, httperrors.NewResourceNotFoundError2(manager.GetSlaveManager().Keyword(), id2) + } else { + return nil, nil, nil, httperrors.NewGeneralError(err) + } } item, err := FetchJointByIds(manager, master.GetId(), slave.GetId(), query) if err != nil { - return nil, nil, nil, err + if errors.Cause(err) == sql.ErrNoRows { + return nil, nil, nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), id1+"-"+id2) + } else { + return nil, nil, nil, httperrors.NewGeneralError(err) + } } return master.(IStandaloneModel), slave.(IStandaloneModel), item, nil } diff --git a/pkg/cloudcommon/db/fetch.go b/pkg/cloudcommon/db/fetch.go index 45417019ee..283bac5c7f 100644 --- a/pkg/cloudcommon/db/fetch.go +++ b/pkg/cloudcommon/db/fetch.go @@ -54,6 +54,7 @@ func FetchJointByIds(manager IJointModelManager, masterId, slaveId string, query } cond := sqlchemy.AND(sqlchemy.Equals(masterField, masterId), sqlchemy.Equals(slaveField, slaveId)) q = q.Filter(cond) + q.DebugQuery() q = manager.FilterByParams(q, query) count, err := q.CountWithError() if err != nil { diff --git a/pkg/compute/models/storagecachedimages.go b/pkg/compute/models/storagecachedimages.go index 1a16e9fc96..68ed681df9 100644 --- a/pkg/compute/models/storagecachedimages.go +++ b/pkg/compute/models/storagecachedimages.go @@ -128,6 +128,9 @@ func (manager *SStoragecachedimageManager) FetchCustomizeColumns( ) []api.StoragecachedimageDetails { rows := make([]api.StoragecachedimageDetails, len(objs)) + storagecacheIds := make([]string, 0) + imageIds := make([]string, 0) + jointRows := manager.SJointResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) scRows := manager.SStoragecacheResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) for i := range rows { @@ -135,7 +138,45 @@ func (manager *SStoragecachedimageManager) FetchCustomizeColumns( JointResourceBaseDetails: jointRows[i], StoragecacheResourceInfo: scRows[i], } - rows[i] = objs[i].(*SStoragecachedimage).getExtraDetails(ctx, rows[i]) + sci := objs[i].(*SStoragecachedimage) + storagecacheIds = append(storagecacheIds, sci.StoragecacheId) + imageIds = append(imageIds, sci.CachedimageId) + } + + cachedImages := make(map[string]SCachedimage) + err := db.FetchModelObjectsByIds(CachedimageManager, "id", imageIds, &cachedImages) + if err != nil { + log.Errorf("db.FetchModelObjectsByIds fail %s", err) + } + cdromRefs, err := manager.fetchCdromReferenceCounts(storagecacheIds, imageIds) + if err != nil { + log.Errorf("manager.fetchCdromReferenceCounts fail %s", err) + } + diskRefs, err := manager.fetchDiskReferenceCounts(storagecacheIds, imageIds) + if err != nil { + log.Errorf("manager.fetchDiskReferenceCounts fail %s", err) + } + + for i := range rows { + sci := objs[i].(*SStoragecachedimage) + if cachedImages != nil { + if cachedImage, ok := cachedImages[sci.CachedimageId]; ok { + rows[i].Cachedimage = cachedImage.Name + rows[i].Image = cachedImage.Name + rows[i].Size = cachedImage.Size + } + } + if cdromRefs != nil { + if refMap, ok := cdromRefs[sci.StoragecacheId]; ok { + rows[i].CdromReference = refMap[sci.CachedimageId] + } + } + if diskRefs != nil { + if refMap, ok := diskRefs[sci.StoragecacheId]; ok { + rows[i].DiskReference = refMap[sci.CachedimageId] + rows[i].Reference = rows[i].CdromReference + rows[i].DiskReference + } + } } return rows @@ -149,7 +190,7 @@ func (self *SStoragecachedimage) GetCachedimage() *SCachedimage { return nil } -func (self *SStoragecachedimage) getExtraDetails(ctx context.Context, out api.StoragecachedimageDetails) api.StoragecachedimageDetails { +/*func (self *SStoragecachedimage) getExtraDetails(ctx context.Context, out api.StoragecachedimageDetails) api.StoragecachedimageDetails { storagecache := self.GetStoragecache() if storagecache != nil { // out.Storagecache = storagecache.Name @@ -176,6 +217,54 @@ func (self *SStoragecachedimage) getExtraDetails(ctx context.Context, out api.St } out.Reference, _ = self.getReferenceCount() return out +}*/ + +func (manager *SStoragecachedimageManager) fetchCdromReferenceCounts(storagecacheIds []string, imageIds []string) (map[string]map[string]int, error) { + q := GuestcdromManager.Query() + guests := GuestManager.Query().SubQuery() + hostStorages := HoststorageManager.Query().SubQuery() + storages := StorageManager.Query().SubQuery() + + q = q.Join(guests, sqlchemy.Equals(q.Field("id"), guests.Field("id"))) + q = q.Join(hostStorages, sqlchemy.Equals(guests.Field("host_id"), hostStorages.Field("host_id"))) + q = q.Join(storages, sqlchemy.Equals(hostStorages.Field("storage_id"), storages.Field("id"))) + + q = q.GroupBy(q.Field("image_id")) + q = q.GroupBy(storages.Field("storagecache_id")) + + q = q.Filter(sqlchemy.In(q.Field("image_id"), imageIds)) + q = q.Filter(sqlchemy.In(storages.Field("storagecache_id"), storagecacheIds)) + + q = q.AppendField(sqlchemy.COUNT("ref_count")) + q = q.AppendField(q.Field("image_id")) + q = q.AppendField(storages.Field("storagecache_id")) + + return manager.fetchRefCount(q) +} + +func (maanger *SStoragecachedimageManager) fetchRefCount(q *sqlchemy.SQuery) (map[string]map[string]int, error) { + results := []struct { + RefCount int `json:"ref_count"` + ImageId string `json:"image_id"` + StoragecacheId string `json:"storagecache_id"` + }{} + + q.DebugQuery() + + err := q.All(&results) + if err != nil { + return nil, errors.Wrap(err, "Query") + } + + ret := make(map[string]map[string]int) + for _, r := range results { + if _, ok := ret[r.StoragecacheId]; !ok { + ret[r.StoragecacheId] = make(map[string]int) + } + ret[r.StoragecacheId][r.ImageId] = r.RefCount + } + + return ret, nil } func (self *SStoragecachedimage) getCdromReferenceCount() (int, error) { @@ -188,6 +277,27 @@ func (self *SStoragecachedimage) getCdromReferenceCount() (int, error) { return q.CountWithError() } +func (manager *SStoragecachedimageManager) fetchDiskReferenceCounts(storagecacheIds, imageIds []string) (map[string]map[string]int, error) { + disks := DiskManager.Query().SubQuery() + storages := StorageManager.Query().SubQuery() + + q := disks.Query() + + q = q.Join(storages, sqlchemy.Equals(disks.Field("storage_id"), storages.Field("id"))) + + q = q.GroupBy(disks.Field("template_id")) + q = q.GroupBy(storages.Field("storagecache_id")) + + q = q.Filter(sqlchemy.In(disks.Field("template_id"), imageIds)) + q = q.Filter(sqlchemy.In(storages.Field("storagecache_id"), storagecacheIds)) + + q = q.AppendField(sqlchemy.COUNT("ref_count")) + q = q.AppendField(disks.Field("template_id").Label("image_id")) + q = q.AppendField(storages.Field("storagecache_id")) + + return manager.fetchRefCount(q) +} + func (self *SStoragecachedimage) getDiskReferenceCount() (int, error) { guestdisks := GuestdiskManager.Query().SubQuery() disks := DiskManager.Query().SubQuery() diff --git a/pkg/compute/models/storagecacheresource.go b/pkg/compute/models/storagecacheresource.go index b3f22a550b..143d4d2353 100644 --- a/pkg/compute/models/storagecacheresource.go +++ b/pkg/compute/models/storagecacheresource.go @@ -87,6 +87,27 @@ func (manager *SStoragecacheResourceBaseManager) FetchCustomizeColumns( return nil } + storageMap := make(map[string][]api.StorageInfo, 0) + { + q := StorageManager.Query("id", "name", "storage_type", "medium_type", "storagecache_id", "zone_id").In("storagecache_id", storagecacheIds) + zones := ZoneManager.Query().SubQuery() + q = q.Join(zones, sqlchemy.Equals(q.Field("zone_id"), zones.Field("id"))) + q = q.AppendField(zones.Field("name").Label("zone")) + + storages := make([]struct { + api.StorageInfo + StoragecacheId string `json:"storagecache_id"` + }, 0) + err := q.All(&storages) + if err != nil { + log.Errorf("Storage Info Query query fail %s", err) + } else { + for _, si := range storages { + storageMap[si.StoragecacheId] = append(storageMap[si.StoragecacheId], si.StorageInfo) + } + } + } + managerList := make([]interface{}, len(rows)) for i := range rows { @@ -96,6 +117,12 @@ func (manager *SStoragecacheResourceBaseManager) FetchCustomizeColumns( rows[i].Storagecache = storagecache.Name rows[i].ManagerId = storagecache.ManagerId } + if info, ok := storageMap[storagecacheIds[i]]; ok { + rows[i].StorageInfo = info + for i := range info { + rows[i].Storages = append(rows[i].Storages, info[i].Name) + } + } managerList[i] = &SManagedResourceBase{rows[i].ManagerId} } diff --git a/pkg/hostman/storageman/imagecache_local.go b/pkg/hostman/storageman/imagecache_local.go index 6626612fc7..1577e73233 100644 --- a/pkg/hostman/storageman/imagecache_local.go +++ b/pkg/hostman/storageman/imagecache_local.go @@ -29,6 +29,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -295,7 +296,7 @@ func (l *SLocalImageCache) Remove(ctx context.Context) error { go func() { _, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx), l.Manager.GetId(), l.imageId, nil) - if err != nil { + if err != nil && httputils.ErrorCode(err) != 404 { log.Errorf("Fail to delete host cached image %s at %s: %s", l.imageId, l.Manager.GetId(), err) } }() diff --git a/pkg/hostman/storageman/imagecache_lvm.go b/pkg/hostman/storageman/imagecache_lvm.go index ebbf5dd511..f17ad7741e 100644 --- a/pkg/hostman/storageman/imagecache_lvm.go +++ b/pkg/hostman/storageman/imagecache_lvm.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/hostutils" @@ -151,7 +152,7 @@ func (c *SLVMImageCache) Remove(ctx context.Context) error { go func() { _, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx), c.Manager.GetId(), c.imageId, nil) - if err != nil { + if err != nil && httputils.ErrorCode(err) != 404 { log.Errorf("Fail to delete host cached image: %s", err) } }() diff --git a/pkg/hostman/storageman/imagecache_rbd.go b/pkg/hostman/storageman/imagecache_rbd.go index 39cd3f4a49..7c573d3eac 100644 --- a/pkg/hostman/storageman/imagecache_rbd.go +++ b/pkg/hostman/storageman/imagecache_rbd.go @@ -22,6 +22,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/hostutils" @@ -123,7 +124,7 @@ func (r *SRbdImageCache) Remove(ctx context.Context) error { go func() { _, err := modules.Storagecachedimages.Detach(hostutils.GetComputeSession(ctx), r.Manager.GetId(), r.imageId, nil) - if err != nil { + if err != nil && httputils.ErrorCode(err) != 404 { log.Errorf("Fail to delete host cached image: %s", err) } }()