opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8440 from rainzm/cachedimage/fasterlist

Browse Source 
refactor(region): faster list for cachedimage
This commit is contained in:
yunion-ci-robot
2020-10-22 21:24:14 +08:00
committed by GitHub
parent 963287b563 bd4851c9f9
commit ae9d1b6ba6
2 changed files with 72 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/compute/models/cachedimages.go
+42 -69
View File
@@ -684,20 +684,6 @@ func (manager *SCachedimageManager) ListItemFilter(
return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter")
}
q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery {
cachedImages := CachedimageManager.Query().SubQuery()
storagecachedImages := StoragecachedimageManager.Query().SubQuery()
storageCaches := StoragecacheManager.Query().SubQuery()
subq := cachedImages.Query(cachedImages.Field("id"))
subq = subq.Join(storagecachedImages, sqlchemy.Equals(cachedImages.Field("id"), storagecachedImages.Field("cachedimage_id")))
subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
return subq
})
if err != nil {
return nil, errors.Wrap(err, "managedResourceFilterByAccount")
}
q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
if err != nil {
return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
@@ -708,74 +694,61 @@ func (manager *SCachedimageManager) ListItemFilter(
return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter")
}
q, err = managedResourceFilterByRegion(q, query.RegionalFilterListInput, "id", func() *sqlchemy.SQuery {
{
storagecachedImages := StoragecachedimageManager.Query().SubQuery()
storageCaches := StoragecacheManager.Query().SubQuery()
storages := StorageManager.Query().SubQuery()
var storages *sqlchemy.SSubQuery
if query.Valid == nil {
storages = StorageManager.Query().SubQuery()
} else if *query.Valid {
storages = StorageManager.Query().In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled").SubQuery()
} else {
stroage := StorageManager.Query()
storages = stroage.Filter(sqlchemy.OR(sqlchemy.NotIn(stroage.Field("status"), []string{}), sqlchemy.IsFalse(stroage.Field("enabled")))).SubQuery()
}
zones := ZoneManager.Query().SubQuery()
subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
subq = subq.Join(zones, sqlchemy.Equals(storages.Field("zone_id"), zones.Field("id")))
subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
return subq
})
if err != nil {
return nil, errors.Wrap(err, "managedResourceFilterByRegion")
}
q, err = managedResourceFilterByZone(q, query.ZonalFilterListInput, "id", func() *sqlchemy.SQuery {
storagecachedImages := StoragecachedimageManager.Query().SubQuery()
storageCaches := StoragecacheManager.Query().SubQuery()
storages := StorageManager.Query().SubQuery()
subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
if len(query.HostSchedtagId) > 0 {
schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
if err != nil {
if errors.Cause(err) == sql.ErrNoRows {
return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
} else {
return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
}
}
hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
}
subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
return subq
})
if err != nil {
return nil, errors.Wrap(err, "managedResourceFilterByZone")
subq, err = managedResourceFilterByAccount(subq, query.ManagedResourceListInput, "", nil)
if err != nil {
return nil, errors.Wrap(err, "managedResourceFilterByAccount")
}
subq, err = _managedResourceFilterByRegion(subq, query.RegionalFilterListInput)
if err != nil {
return nil, errors.Wrap(err, "_managedResourceFilterByRegion")
}
subq, err = _managedResourceFilterByZone(subq, query.ZonalFilterListInput)
if err != nil {
return nil, errors.Wrap(err, "_managedResourceFilterByZone")
}
q = q.In("id", subq)
}
if len(query.ImageType) > 0 {
q = q.In("image_type", query.ImageType)
}
if len(query.HostSchedtagId) > 0 {
schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
if err != nil {
if errors.Cause(err) == sql.ErrNoRows {
return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
} else {
return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
}
}
subq := StoragecachedimageManager.Query("cachedimage_id")
storages := StorageManager.Query("id", "storagecache_id").SubQuery()
hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
subq = subq.Join(storages, sqlchemy.Equals(storages.Field("storagecache_id"), subq.Field("storagecache_id")))
subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
q = q.In("id", subq.SubQuery())
}
if query.Valid != nil {
storagecachedImages := StoragecachedimageManager.Query().SubQuery()
storageCaches := StoragecacheManager.Query().SubQuery()
storages := StorageManager.Query().In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled").SubQuery()
subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
if *query.Valid {
q = q.In("id", subq.SubQuery())
} else {
q = q.NotIn("id", subq.SubQuery())
}
q = q.Equals("image_type", query.ImageType)
}
return q, nil
pkg/compute/models/managedresource.go
+30 -31
View File
@@ -557,7 +557,7 @@ func _managedResourceFilterByAccount(managerIdFieldName string, q *sqlchemy.SQue
return q, nil
}
func managedResourceFilterByZone(q *sqlchemy.SQuery, query api.ZonalFilterListInput, filterField string, subqFunc func() *sqlchemy.SQuery) (*sqlchemy.SQuery, error) {
func _managedResourceFilterByZone(q *sqlchemy.SQuery, query api.ZonalFilterListInput) (*sqlchemy.SQuery, error) {
zoneList := query.ZoneList()
if len(query.ZoneIds) >= 1 {
zoneQ := ZoneManager.Query("id")
@@ -565,58 +565,57 @@ func managedResourceFilterByZone(q *sqlchemy.SQuery, query api.ZonalFilterListIn
sqlchemy.In(zoneQ.Field("id"), zoneList),
sqlchemy.In(zoneQ.Field("name"), zoneList),
))
if len(filterField) == 0 {
q = q.Filter(sqlchemy.In(q.Field("zone_id"), zoneQ.SubQuery()))
} else {
sq := subqFunc()
sq = sq.Filter(sqlchemy.In(sq.Field("zone_id"), zoneQ.SubQuery()))
q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
}
q = q.Filter(sqlchemy.In(q.Field("zone_id"), zoneQ.SubQuery()))
} else if len(query.ZoneId) > 0 {
zoneObj, _, err := ValidateZoneResourceInput(nil, query.ZoneResourceInput)
if err != nil {
return nil, errors.Wrap(err, "ValidateZoneResourceInput")
}
if len(filterField) == 0 {
q = q.Filter(sqlchemy.Equals(q.Field("zone_id"), zoneObj.GetId()))
} else {
sq := subqFunc()
sq = sq.Filter(sqlchemy.Equals(sq.Field("zone_id"), zoneObj.GetId()))
q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
}
q = q.Filter(sqlchemy.Equals(q.Field("zone_id"), zoneObj.GetId()))
}
return q, nil
}
func managedResourceFilterByRegion(q *sqlchemy.SQuery, query api.RegionalFilterListInput, filterField string, subqFunc func() *sqlchemy.SQuery) (*sqlchemy.SQuery, error) {
func managedResourceFilterByZone(q *sqlchemy.SQuery, query api.ZonalFilterListInput, filterField string, subqFunc func() *sqlchemy.SQuery) (*sqlchemy.SQuery, error) {
return filterFieldFromSubQuery(q, filterField, subqFunc, func(s *sqlchemy.SQuery) (*sqlchemy.SQuery, error) {
return _managedResourceFilterByZone(s, query)
})
}
func filterFieldFromSubQuery(q *sqlchemy.SQuery, filterField string, subqFunc func() *sqlchemy.SQuery, filterAdd func(query *sqlchemy.SQuery) (*sqlchemy.SQuery, error)) (*sqlchemy.SQuery, error) {
if len(filterField) == 0 {
return filterAdd(q)
}
sq, err := filterAdd(subqFunc())
if err != nil {
return nil, err
}
q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
return q, nil
}
func _managedResourceFilterByRegion(q *sqlchemy.SQuery, query api.RegionalFilterListInput) (*sqlchemy.SQuery, error) {
regionStr := query.CloudregionId
if len(regionStr) > 0 {
regionObj, _, err := ValidateCloudregionResourceInput(nil, query.CloudregionResourceInput)
if err != nil {
return nil, errors.Wrap(err, "ValidateCloudregionResourceInput")
}
if len(filterField) == 0 {
q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), regionObj.GetId()))
} else {
sq := subqFunc()
sq = sq.Filter(sqlchemy.Equals(sq.Field("cloudregion_id"), regionObj.GetId()))
q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
}
q = q.Filter(sqlchemy.Equals(q.Field("cloudregion_id"), regionObj.GetId()))
}
if len(query.City) > 0 {
subq := CloudregionManager.Query("id").Equals("city", query.City).SubQuery()
if len(filterField) == 0 {
q = q.Filter(sqlchemy.In(q.Field("cloudregion_id"), subq))
} else {
sq := subqFunc()
sq = sq.Filter(sqlchemy.In(sq.Field("cloudregion_id"), subq))
q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
}
q = q.Filter(sqlchemy.In(q.Field("cloudregion_id"), subq))
}
return q, nil
}
func managedResourceFilterByRegion(q *sqlchemy.SQuery, query api.RegionalFilterListInput, filterField string, subqFunc func() *sqlchemy.SQuery) (*sqlchemy.SQuery, error) {
return filterFieldFromSubQuery(q, filterField, subqFunc, func(s *sqlchemy.SQuery) (*sqlchemy.SQuery, error) {
return _managedResourceFilterByRegion(s, query)
})
}
func _filterByCloudType(managerIdFieldName string, q *sqlchemy.SQuery, input api.ManagedResourceListInput, filterField string, subqFunc func() *sqlchemy.SQuery) *sqlchemy.SQuery {
cloudEnvStr := input.CloudEnv