From 34ed406dddbd325bd9fdd08fa217fa2de6bc7f6d Mon Sep 17 00:00:00 2001 From: TangBin Date: Thu, 13 Dec 2018 17:44:48 +0800 Subject: [PATCH 1/3] add clean invalid sku job --- pkg/compute/models/skus.go | 37 ++++++++++++++++++++++++++++++++++ pkg/compute/service/service.go | 1 + pkg/compute/skus/skus.go | 10 +++++++++ 3 files changed, 48 insertions(+) diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index f3ef8529ec..26ae0972b0 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -536,3 +536,40 @@ func (manager *SServerSkuManager) GetSkuCountByProvider(provider string) int { return q.Count() } + +// 删除表中zone not found的记录 +func (manager *SServerSkuManager) PendingDeleteInvalidSku() error { + zones := make([]SZone, 0) + err := ZoneManager.Query("id").Distinct().All(&zones) + if err != nil { + log.Errorf(err.Error()) + return httperrors.NewInternalServerError("query zone list failed.") + } + + zoneIds := []string{} + for _, zone := range zones { + zoneIds = append(zoneIds, zone.Id) + } + + skus := make([]SServerSku, 0) + q := manager.Query() + err = q.NotIn("zone_id", zoneIds).All(&skus) + if err != nil { + log.Errorf(err.Error()) + return httperrors.NewInternalServerError("query sku list failed.") + } + + for i := range skus { + sku := skus[i] + _, err = manager.TableSpec().Update(&sku, func() error { + return sku.MarkDelete() + }) + + if err != nil { + log.Errorf(err.Error()) + return httperrors.NewInternalServerError("delete sku %s failed.", sku.Id) + } + } + + return nil +} diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index bd4623bd21..bc41ecc8b3 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -63,6 +63,7 @@ func StartService() { cron.AddJob2("AutoDiskSnapshot", options.Options.AutoSnapshotDay, options.Options.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot, false) cron.AddJob2("SyncSkus", options.Options.SyncSkusDay, options.Options.SyncSkusHour, 0, 0, skus.SyncSkus, true) + cron.AddJob2("CleanInvalidSkus", options.Options.SyncSkusDay, options.Options.SyncSkusHour, 0, 0, skus.DeleteInvalidSkus, true) // skus只在启动时进行清理 cron.Start() defer cron.Stop() diff --git a/pkg/compute/skus/skus.go b/pkg/compute/skus/skus.go index 58e675ffe7..5ec475b3e9 100644 --- a/pkg/compute/skus/skus.go +++ b/pkg/compute/skus/skus.go @@ -306,6 +306,7 @@ func (self *SkusZoneList) SyncToLocalDB() error { return err } +// 全量同步sku列表. func SyncSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { if isStart { if models.ServerSkuManager.GetSkuCountByProvider("") > 0 { @@ -322,6 +323,7 @@ func SyncSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bo } } +// 同步指定provider sku列表 func SyncSkusByProviderIds(providerIds []string) error { skus := SkusZoneList{} log.Debugf("SyncSkusByProviderIds %s", providerIds) @@ -335,3 +337,11 @@ func SyncSkusByProviderIds(providerIds []string) error { return nil } + +// 伪删除无效的sku.只在启动进程时进行清理。 +func DeleteInvalidSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { + if isStart { + log.Debugf("DeleteInvalidSkus in processing...") + models.ServerSkuManager.PendingDeleteInvalidSku() + } +} From cc6d3a5991239e917a2bd606a743469ee36a76a8 Mon Sep 17 00:00:00 2001 From: TangBin Date: Thu, 13 Dec 2018 19:49:30 +0800 Subject: [PATCH 2/3] delete invalid sku while sync skus --- pkg/compute/service/service.go | 1 - pkg/compute/skus/skus.go | 12 ++++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index bc41ecc8b3..bd4623bd21 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -63,7 +63,6 @@ func StartService() { cron.AddJob2("AutoDiskSnapshot", options.Options.AutoSnapshotDay, options.Options.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot, false) cron.AddJob2("SyncSkus", options.Options.SyncSkusDay, options.Options.SyncSkusHour, 0, 0, skus.SyncSkus, true) - cron.AddJob2("CleanInvalidSkus", options.Options.SyncSkusDay, options.Options.SyncSkusHour, 0, 0, skus.DeleteInvalidSkus, true) // skus只在启动时进行清理 cron.Start() defer cron.Stop() diff --git a/pkg/compute/skus/skus.go b/pkg/compute/skus/skus.go index 5ec475b3e9..bbbb4ae511 100644 --- a/pkg/compute/skus/skus.go +++ b/pkg/compute/skus/skus.go @@ -321,6 +321,10 @@ func SyncSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bo if e := skus.SyncToLocalDB(); e != nil { log.Errorf("SyncSkus sync to local db failed, %s", e.Error()) } + + // 清理无效的sku + log.Debugf("DeleteInvalidSkus in processing...") + models.ServerSkuManager.PendingDeleteInvalidSku() } // 同步指定provider sku列表 @@ -337,11 +341,3 @@ func SyncSkusByProviderIds(providerIds []string) error { return nil } - -// 伪删除无效的sku.只在启动进程时进行清理。 -func DeleteInvalidSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { - if isStart { - log.Debugf("DeleteInvalidSkus in processing...") - models.ServerSkuManager.PendingDeleteInvalidSku() - } -} From 17ec3fecb06dd64dcd8bd57e5e412fc710b06d54 Mon Sep 17 00:00:00 2001 From: TangBin Date: Fri, 14 Dec 2018 09:52:22 +0800 Subject: [PATCH 3/3] skus delete query fix --- pkg/compute/models/skus.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index 26ae0972b0..420632f42d 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -539,21 +539,11 @@ func (manager *SServerSkuManager) GetSkuCountByProvider(provider string) int { // 删除表中zone not found的记录 func (manager *SServerSkuManager) PendingDeleteInvalidSku() error { - zones := make([]SZone, 0) - err := ZoneManager.Query("id").Distinct().All(&zones) - if err != nil { - log.Errorf(err.Error()) - return httperrors.NewInternalServerError("query zone list failed.") - } - - zoneIds := []string{} - for _, zone := range zones { - zoneIds = append(zoneIds, zone.Id) - } - + sq := ZoneManager.Query("id").Distinct().SubQuery() skus := make([]SServerSku, 0) q := manager.Query() - err = q.NotIn("zone_id", zoneIds).All(&skus) + q = q.NotIn("zone_id", sq).IsNotEmpty("zone_id") + err := db.FetchModelObjects(manager, q, &skus) if err != nil { log.Errorf(err.Error()) return httperrors.NewInternalServerError("query sku list failed.")