From 7c3929bc5ef5cfdc485f6d424a257287b11b9015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Fri, 19 Dec 2025 12:42:18 +0800 Subject: [PATCH] fix(region): allow clean history data (#23950) --- cmd/climc/shell/misc/history_data_clean.go | 37 +++++++++++ pkg/cloudcommon/db/history_data_clean.go | 71 +++++++++++++++++++++ pkg/cloudcommon/db/jointbase.go | 68 ++++++++++++++++++++ pkg/cloudcommon/db/standalone_anon.go | 67 +++++++++++++++++++ pkg/compute/service/handlers.go | 1 + pkg/mcclient/modulebase/historydataclean.go | 42 ++++++++++++ pkg/mcclient/modules/historydataclean.go | 26 ++++++++ 7 files changed, 312 insertions(+) create mode 100644 cmd/climc/shell/misc/history_data_clean.go create mode 100644 pkg/cloudcommon/db/history_data_clean.go create mode 100644 pkg/mcclient/modulebase/historydataclean.go create mode 100644 pkg/mcclient/modules/historydataclean.go diff --git a/cmd/climc/shell/misc/history_data_clean.go b/cmd/climc/shell/misc/history_data_clean.go new file mode 100644 index 0000000000..5da56b1d9c --- /dev/null +++ b/cmd/climc/shell/misc/history_data_clean.go @@ -0,0 +1,37 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package misc + +import ( + "fmt" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + type HistoryDataCleanOptions struct { + SERVICE string `help:"Service type"` + Day *int `default:"30"` + } + R(&HistoryDataCleanOptions{}, "history-data-clean", "clean history data", func(s *mcclient.ClientSession, args *HistoryDataCleanOptions) error { + body, err := modules.HistoryDataClean(s, args.SERVICE, args.Day) + if err != nil { + return err + } + fmt.Println(body.PrettyString()) + return nil + }) +} diff --git a/pkg/cloudcommon/db/history_data_clean.go b/pkg/cloudcommon/db/history_data_clean.go new file mode 100644 index 0000000000..9af341b5dc --- /dev/null +++ b/pkg/cloudcommon/db/history_data_clean.go @@ -0,0 +1,71 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +import ( + "context" + "fmt" + "net/http" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +func AddHistoryDataCleanHandler(prefix string, app *appsrv.Application) { + prefix = fmt.Sprintf("%s/history-data-clean", prefix) + app.AddHandler2("POST", prefix, auth.Authenticate(historyDataCleanHandler), nil, "history_data_clean", nil) +} + +func historyDataCleanHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + userCred := fetchUserCredential(ctx) + if !userCred.HasSystemAdminPrivilege() { + httperrors.ForbiddenError(ctx, w, "only sysadmin can clean history data") + return + } + input, err := appsrv.FetchJSON(r) + if err != nil { + httperrors.InputParameterError(ctx, w, "FetchJSON") + return + } + date := time.Now().AddDate(0, -1, 0) + if !gotypes.IsNil(input) && input.Contains("day") { + day, _ := input.Int("day") + date = time.Now().AddDate(0, 0, int(day)*-1) + } + go func() { + for _, manager := range globalTables { + if hM, ok := manager.(IHistoryDataManager); ok { + start := time.Now() + cnt, err := hM.HistoryDataClean(ctx, date) + if err != nil { + log.Errorf("clean %s data error: %v", manager.Keyword(), err) + continue + } + log.Debugf("clean %d %s history data cost %s", cnt, manager.Keyword(), time.Now().Sub(start).Round(time.Second)) + } + } + }() + appsrv.SendJSON(w, jsonutils.Marshal(map[string]string{"status": "ok"})) +} + +type IHistoryDataManager interface { + HistoryDataClean(ctx context.Context, timeBefor time.Time) (int, error) +} diff --git a/pkg/cloudcommon/db/jointbase.go b/pkg/cloudcommon/db/jointbase.go index f9baf10f77..20dbfb345f 100644 --- a/pkg/cloudcommon/db/jointbase.go +++ b/pkg/cloudcommon/db/jointbase.go @@ -16,8 +16,11 @@ package db import ( "context" + "database/sql" "fmt" "reflect" + "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -27,6 +30,7 @@ import ( "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/stringutils2" ) @@ -255,3 +259,67 @@ func (model *SJointResourceBase) ValidateUpdateData( } return input, nil } + +func (manager *SJointResourceBaseManager) HistoryDataClean(ctx context.Context, timeBefor time.Time) (int, error) { + q := manager.RawQuery("row_id").IsTrue("deleted").LE("deleted_at", timeBefor) + rows, err := q.Rows() + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return 0, nil + } + return 0, errors.Wrap(err, "Query") + } + defer rows.Close() + ids := []string{} + for rows.Next() { + var id string + err := rows.Scan(&id) + if err != nil { + return 0, errors.Wrap(err, "rows.Scan") + } + ids = append(ids, id) + } + var purge = func(ids []string) error { + vars := []interface{}{} + placeholders := make([]string, len(ids)) + for i := range placeholders { + placeholders[i] = "?" + vars = append(vars, ids[i]) + } + placeholder := strings.Join(placeholders, ",") + sql := fmt.Sprintf( + "delete from %s where row_id in (%s)", + manager.TableSpec().Name(), placeholder, + ) + lockman.LockRawObject(ctx, manager.Keyword(), "purge") + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "purge") + + _, err = sqlchemy.GetDB().Exec( + sql, vars..., + ) + if err != nil { + return errors.Wrapf(err, strings.ReplaceAll(sql, "?", "%s"), vars...) + } + return nil + } + + var splitByLen = func(data []string, splitLen int) [][]string { + var result [][]string + for i := 0; i < len(data); i += splitLen { + end := i + splitLen + if end > len(data) { + end = len(data) + } + result = append(result, data[i:end]) + } + return result + } + idsArr := splitByLen(ids, 100) + for i := range idsArr { + err = purge(idsArr[i]) + if err != nil { + return 0, err + } + } + return len(ids), nil +} diff --git a/pkg/cloudcommon/db/standalone_anon.go b/pkg/cloudcommon/db/standalone_anon.go index 279187fa09..95da5038d9 100644 --- a/pkg/cloudcommon/db/standalone_anon.go +++ b/pkg/cloudcommon/db/standalone_anon.go @@ -17,8 +17,10 @@ package db import ( "context" "crypto/md5" + "database/sql" "fmt" "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -29,6 +31,7 @@ import ( "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/cloudcommon/consts" + "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" @@ -1054,3 +1057,67 @@ func GetTagValueCountMap( } return valueMap, nil } + +func (manager *SStandaloneAnonResourceBaseManager) HistoryDataClean(ctx context.Context, timeBefor time.Time) (int, error) { + q := manager.RawQuery("id").IsTrue("deleted").LE("deleted_at", timeBefor) + rows, err := q.Rows() + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return 0, nil + } + return 0, errors.Wrap(err, "Query") + } + defer rows.Close() + ids := []string{} + for rows.Next() { + var id string + err := rows.Scan(&id) + if err != nil { + return 0, errors.Wrap(err, "rows.Scan") + } + ids = append(ids, id) + } + var purge = func(ids []string) error { + vars := []interface{}{} + placeholders := make([]string, len(ids)) + for i := range placeholders { + placeholders[i] = "?" + vars = append(vars, ids[i]) + } + placeholder := strings.Join(placeholders, ",") + sql := fmt.Sprintf( + "delete from %s where id in (%s)", + manager.TableSpec().Name(), placeholder, + ) + lockman.LockRawObject(ctx, manager.Keyword(), "purge") + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "purge") + + _, err = sqlchemy.GetDB().Exec( + sql, vars..., + ) + if err != nil { + return errors.Wrapf(err, strings.ReplaceAll(sql, "?", "%s"), vars...) + } + return nil + } + + var splitByLen = func(data []string, splitLen int) [][]string { + var result [][]string + for i := 0; i < len(data); i += splitLen { + end := i + splitLen + if end > len(data) { + end = len(data) + } + result = append(result, data[i:end]) + } + return result + } + idsArr := splitByLen(ids, 100) + for i := range idsArr { + err = purge(idsArr[i]) + if err != nil { + return 0, err + } + } + return len(ids), nil +} diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index bf55a4b717..0566d44a8c 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -38,6 +38,7 @@ func InitHandlers(app *appsrv.Application) { db.RegistUserCredCacheUpdater() db.AddScopeResourceCountHandler("", app) + db.AddHistoryDataCleanHandler("", app) quotas.AddQuotaHandler(&models.QuotaManager.SQuotaBaseManager, "", app) quotas.AddQuotaHandler(&models.RegionQuotaManager.SQuotaBaseManager, "", app) diff --git a/pkg/mcclient/modulebase/historydataclean.go b/pkg/mcclient/modulebase/historydataclean.go new file mode 100644 index 0000000000..a1ed0ae225 --- /dev/null +++ b/pkg/mcclient/modulebase/historydataclean.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modulebase + +import ( + "io" + "strings" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" +) + +func HistoryDataClean(s *mcclient.ClientSession, serviceType string, day *int) (jsonutils.JSONObject, error) { + man := &BaseManager{serviceType: serviceType} + input := jsonutils.NewDict() + if day != nil { + input.Set("day", jsonutils.NewInt(int64(*day))) + } + resp, err := man.rawRequest(s, "POST", "/history-data-clean", nil, strings.NewReader(input.String())) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return jsonutils.Parse(body) +} diff --git a/pkg/mcclient/modules/historydataclean.go b/pkg/mcclient/modules/historydataclean.go new file mode 100644 index 0000000000..d274c6769a --- /dev/null +++ b/pkg/mcclient/modules/historydataclean.go @@ -0,0 +1,26 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" +) + +func HistoryDataClean(s *mcclient.ClientSession, serviceType string, day *int) (jsonutils.JSONObject, error) { + return modulebase.HistoryDataClean(s, serviceType, day) +}