opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-10-29 17:58:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: splitable duplicate records (#15032)

Browse Source 
Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
Jian Qiu
2022-09-23 14:09:00 +08:00
committed by GitHub
co-authored by Qiu Jian
parent 194ab2dc89
commit 8d077e2a88
11 changed files with 127 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/util/splitable/insert.go
+44 -20
View File
@@ -15,6 +15,7 @@
package splitable
import (
"database/sql"
"fmt"
"reflect"
"time"
@@ -26,17 +27,26 @@ import (
"yunion.io/x/sqlchemy"
)
func (t *SSplitTableSpec) Insert(dt interface{}) error {
metas, err := t.GetTableMetas()
if err != nil {
return errors.Wrap(err, "GetTableMeta")
func (t *SSplitTableSpec) getLastTableSpecWithLock(lastDate time.Time) (*sqlchemy.STableSpec, error) {
t.lastTableLock.Lock()
defer t.lastTableLock.Unlock()
now := time.Now()
if t.lastTableSpec != nil && !t.lastTableExpire.IsZero() && t.lastTableExpire.Before(now) {
return t.lastTableSpec, nil
}
var lastDate time.Time
vs := reflectutils.FetchAllStructFieldValueSet(reflect.Indirect(reflect.ValueOf(dt)))
if lastDateV, ok := vs.GetValue(t.dateField); !ok {
return errors.Wrap(errors.ErrInvalidStatus, "no dateField found")
} else {
lastDate = lastDateV.Interface().(time.Time)
lastTableSpec, err := t.getLastTableSpec(lastDate)
if err != nil {
return nil, errors.Wrap(err, "getLastTableSpec")
}
t.lastTableSpec = lastTableSpec
t.lastTableExpire = now.Add(time.Hour * lastTableSpecExpireHours)
return t.lastTableSpec, nil
}
func (t *SSplitTableSpec) getLastTableSpec(lastDate time.Time) (*sqlchemy.STableSpec, error) {
lastMeta, err := t.getTableLastMeta()
if err != nil && errors.Cause(err) != sql.ErrNoRows {
return nil, errors.Wrap(err, "GetTableMeta")
}
var lastRecIndex int64
@@ -44,10 +54,9 @@ func (t *SSplitTableSpec) Insert(dt interface{}) error {
var lastTableSpec *sqlchemy.STableSpec
newMeta := false
if len(metas) > 0 {
lastMeta := metas[len(metas)-1]
if lastMeta != nil {
if !lastMeta.StartDate.IsZero() && lastDate.Sub(lastMeta.StartDate) > t.maxDuration {
lastTable := t.GetTableSpec(lastMeta)
lastTable := t.GetTableSpec(*lastMeta)
ti := lastTable.Instance()
q := ti.Query(sqlchemy.MAX("last_index", ti.Field(t.indexField)), sqlchemy.MAX("last_date", ti.Field(t.dateField)), sqlchemy.COUNT("total"))
r := q.Row()
@@ -55,34 +64,34 @@ func (t *SSplitTableSpec) Insert(dt interface{}) error {
var total uint64
err := r.Scan(&lastRecIndex, &lastRecDateStr, &total)
if err != nil {
return errors.Wrap(err, "scan lastRecIndex and lastRecDate")
return nil, errors.Wrap(err, "scan lastRecIndex and lastRecDate")
}
log.Debugf("lastRecDateStr: %s", lastRecDateStr)
lastRecDate, _ = timeutils.ParseTimeStr(lastRecDateStr)
// seal last meta
_, err = t.metaSpec.Update(&lastMeta, func() error {
_, err = t.metaSpec.Update(lastMeta, func() error {
lastMeta.End = lastRecIndex
lastMeta.EndDate = lastRecDate
lastMeta.Count = total
return nil
})
if err != nil {
return errors.Wrap(err, "Update last meta")
return nil, errors.Wrap(err, "Update last meta")
}
newMeta = true
} else {
if lastMeta.StartDate.IsZero() {
indexCol := t.tableSpec.ColumnSpec(t.indexField)
_, err = t.metaSpec.Update(&lastMeta, func() error {
_, err = t.metaSpec.Update(lastMeta, func() error {
lastMeta.Start = indexCol.AutoIncrementOffset()
lastMeta.StartDate = lastDate
return nil
})
if err != nil {
return errors.Wrap(err, "Update last meta")
return nil, errors.Wrap(err, "Update last meta")
}
}
lastTableSpec = t.GetTableSpec(lastMeta)
lastTableSpec = t.GetTableSpec(*lastMeta)
}
} else {
newMeta = true
@@ -90,9 +99,24 @@ func (t *SSplitTableSpec) Insert(dt interface{}) error {
if newMeta {
lastTableSpec, err = t.newTable(lastRecIndex, lastDate)
if err != nil {
return errors.Wrap(err, "newTable")
return nil, errors.Wrap(err, "newTable")
}
}
return lastTableSpec, nil
}
func (t *SSplitTableSpec) Insert(dt interface{}) error {
var lastDate time.Time
vs := reflectutils.FetchAllStructFieldValueSet(reflect.Indirect(reflect.ValueOf(dt)))
if lastDateV, ok := vs.GetValue(t.dateField); !ok {
return errors.Wrap(errors.ErrInvalidStatus, "no dateField found")
} else {
lastDate = lastDateV.Interface().(time.Time)
}
lastTableSpec, err := t.getLastTableSpecWithLock(lastDate)
if err != nil {
return errors.Wrap(err, "getLastTableSpec")
}
return lastTableSpec.Insert(dt)
}
pkg/util/splitable/metadata.go
+8 -1
View File
@@ -35,10 +35,17 @@ type STableMetadata struct {
CreatedAt time.Time `nullable:"false" created_at:"true"`
}
func (spec *SSplitTableSpec) getTableLastMeta() (*STableMetadata, error) {
return spec.GetTableMetaByTime(time.Time{})
}
func (spec *SSplitTableSpec) GetTableMetaByTime(recordTime time.Time) (*STableMetadata, error) {
q := spec.metaSpec.Query().Desc("id").IsFalse("deleted")
if !recordTime.IsZero() {
q = q.LE("start_date", recordTime)
}
meta := new(STableMetadata)
err := q.LE("start_date", recordTime).First(meta)
err := q.First(meta)
return meta, err
}
pkg/util/splitable/purge.go
+11 -3
View File
@@ -21,7 +21,6 @@ import (
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
"yunion.io/x/sqlchemy"
)
func (t *SSplitTableSpec) Purge(tables []string) ([]string, error) {
@@ -35,12 +34,21 @@ func (t *SSplitTableSpec) Purge(tables []string) ([]string, error) {
if t.maxSegments >= len(metas) && len(tables) == 0 {
return nil, nil
}
metaMax := len(metas)
if len(tables) == 0 {
// keep maxSegments if no table specified
metaMax -= t.maxSegments
} else {
// cannot delete the last one
metaMax -= 1
}
ret := []string{}
for i := 0; i < len(metas); i += 1 {
for i := 0; i < metaMax; i += 1 {
if len(tables) == 0 || utils.IsInStringArray(metas[i].Table, tables) {
dropSQL := fmt.Sprintf("DROP TABLE `%s`", metas[i].Table)
log.Infof("Ready to drop table: %s", dropSQL)
_, err := sqlchemy.Exec(dropSQL)
tblSpec := t.GetTableSpec(metas[i])
err := tblSpec.Drop()
if err != nil {
return ret, errors.Wrap(err, "sqlchemy.Exec")
}
pkg/util/splitable/splitable.go
+30
View File
@@ -18,6 +18,7 @@ import (
"database/sql"
"fmt"
"reflect"
"sync"
"time"
"yunion.io/x/pkg/errors"
@@ -33,8 +34,16 @@ type SSplitTableSpec struct {
metaSpec *sqlchemy.STableSpec
maxDuration time.Duration
maxSegments int
lastTableSpec *sqlchemy.STableSpec
lastTableLock *sync.Mutex
lastTableExpire time.Time
}
const (
lastTableSpecExpireHours = 1 // 1 hour
)
func (t *SSplitTableSpec) DataType() reflect.Type {
return t.tableSpec.DataType()
}
@@ -112,6 +121,25 @@ func (t *SSplitTableSpec) Fetch(dt interface{}) error {
return sql.ErrNoRows
}
func (t *SSplitTableSpec) Drop() error {
metas, err := t.GetTableMetas()
if err != nil {
return errors.Wrap(err, "GetTableMetas")
}
for _, meta := range metas {
ts := t.GetTableSpec(meta)
err := ts.Drop()
if err != nil {
return errors.Wrapf(err, "Drop %s", meta.Table)
}
}
err = t.metaSpec.Drop()
if err != nil {
return errors.Wrap(err, "Drop Meta")
}
return nil
}
func NewSplitTableSpec(s interface{}, name string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) (*SSplitTableSpec, error) {
spec := sqlchemy.NewTableSpecFromStructWithDBName(s, name, dbName)
/*indexCol := spec.ColumnSpec(indexField)
@@ -142,6 +170,8 @@ func NewSplitTableSpec(s interface{}, name string, indexField string, dateField
metaSpec: metaSpec,
maxDuration: maxDuration,
maxSegments: maxSegments,
lastTableLock: &sync.Mutex{},
}
return sts, nil