fix(account): prevent stale debt deletion after recharge (#7230)

* fix(account): prevent stale debt deletion after recharge

* style(account): satisfy golangci-lint formatting

* fix(account): improve debt deletion diagnostics
This commit is contained in:
Yun Pan
2026-08-18 11:26:51 +08:00
committed by GitHub
parent a04e0eb544
commit a706e3297a
9 changed files with 382 additions and 20 deletions
+62 -12
View File
@@ -720,9 +720,50 @@ func (r *DebtReconciler) syncFinalDeletionDebtNamespacesForUser(
ctx context.Context,
userUID uuid.UUID,
) error {
mutex, err := r.getUserMutex(userUID)
if err != nil {
return err
}
mutex.Lock()
defer mutex.Unlock()
// The hourly query can race with a successful recharge. Recompute the
// current debt state while holding the same lock as normal refreshes before
// issuing another destructive cleanup request.
var debt types.Debt
if err := r.AccountV2.GetGlobalDB().
Where("user_uid = ?", userUID).
First(&debt).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return fmt.Errorf("failed to reload debt %s: %w", userUID, err)
}
if debt.AccountDebtStatus != types.FinalDeletionPeriod {
return nil
}
if err := r.refreshDebtStatus(userUID, true); err != nil {
return fmt.Errorf("failed to refresh debt status before final deletion replay: %w", err)
}
if err := r.AccountV2.GetGlobalDB().
Where("user_uid = ?", userUID).
First(&debt).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return fmt.Errorf("failed to reload debt %s after refresh: %w", userUID, err)
}
if debt.AccountDebtStatus != types.FinalDeletionPeriod {
return nil
}
req := finalDeletionDebtNamespaceFlushReq(userUID)
req.ReplayFinalDeletion = true
return r.sendFlushDebtResourceStatusRequestWithContext(
ctx,
finalDeletionDebtNamespaceFlushReq(userUID),
req,
)
}
@@ -1055,10 +1096,11 @@ func (r *DebtReconciler) SendUserDebtMsg(
}
type AdminFlushResourceStatusReq struct {
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
ReplayFinalDeletion bool `json:"replayFinalDeletion,omitempty" bson:"replayFinalDeletion,omitempty"`
}
// TODO flush desktop message (send or read) && flush resource quota (suspend or resume or delete)
@@ -1213,6 +1255,18 @@ func (r *DebtReconciler) retryFailedUsers() {
}
}
func (r *DebtReconciler) getUserMutex(userUID uuid.UUID) (*sync.Mutex, error) {
if r.userLocks == nil {
return nil, errors.New("user locks are not initialized")
}
lock, _ := r.userLocks.LoadOrStore(userUID, &sync.Mutex{})
mutex, ok := lock.(*sync.Mutex)
if !ok {
return nil, fmt.Errorf("invalid mutex for user %s", userUID)
}
return mutex, nil
}
// Parallel processing of user debt status, the same user simultaneously through the lock to implement a debt refresh processing.
func (r *DebtReconciler) processUsersInParallel(users []uuid.UUID) {
var (
@@ -1226,13 +1280,9 @@ func (r *DebtReconciler) processUsersInParallel(users []uuid.UUID) {
go func(u uuid.UUID) {
defer wg.Done()
defer func() { <-semaphore }()
lock, _ := r.userLocks.LoadOrStore(u, &sync.Mutex{})
mutex, ok := lock.(*sync.Mutex)
if !ok {
r.Error(
fmt.Errorf("invalid mutex for user %s", u),
"failed to load user mutex",
)
mutex, err := r.getUserMutex(u)
if err != nil {
r.Error(err, "failed to load user mutex", "userUID", u)
return
}
if !mutex.TryLock() {
@@ -29,4 +29,17 @@ func TestFinalDeletionDebtNamespaceFlushReq(t *testing.T) {
req.CurrentDebtStatus,
)
}
if req.ReplayFinalDeletion {
t.Fatal("normal final deletion request should not be marked as replay")
}
}
func TestFinalDeletionDebtNamespaceReplayReq(t *testing.T) {
userUID := uuid.New()
req := finalDeletionDebtNamespaceFlushReq(userUID)
req.ReplayFinalDeletion = true
if !req.ReplayFinalDeletion {
t.Fatal("final deletion replay request should be marked as replay")
}
}
@@ -74,6 +74,8 @@ const (
DeployPVCResizeKey = "deploy.cloud.sealos.io/resize"
)
var errFinalDeletionCancelled = errors2.New("final deletion cancelled by namespace status")
//+kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=namespaces/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=core,resources=namespaces/finalizers,verbs=update
@@ -146,6 +148,9 @@ func (r *NamespaceReconciler) Reconcile(
// auxiliary function handles resource operations
performAction := func(action func(context.Context, string) error, actionName string) (ctrl.Result, error) {
if err := action(ctx, req.Name); err != nil {
if errors2.Is(err, errFinalDeletionCancelled) {
return ctrl.Result{}, err
}
logger.Error(err, actionName+" namespace resources failed")
return ctrl.Result{
Requeue: actionName == deleteConst,
@@ -379,6 +384,10 @@ func (r *NamespaceReconciler) Reconcile(
if t.condition() {
if t.action != nil {
if result, err := performAction(t.action, t.actionName); err != nil {
if errors2.Is(err, errFinalDeletionCancelled) {
logger.Info("final deletion cancelled by current namespace status")
return ctrl.Result{}, nil
}
return result, err
}
}
@@ -441,10 +450,60 @@ func (r *NamespaceReconciler) deleteBackup(ctx context.Context, namespace string
Version: "v1alpha1",
Resource: "backups",
}
return deleteResourceListAndWait(ctx, r.dynamicClient, gvr, namespace, r.deleteBackupSemaphore)
return deleteResourceListAndWait(
ctx,
r.dynamicClient,
gvr,
namespace,
r.deleteBackupSemaphore,
func(ctx context.Context) error {
return r.ensureFinalDeletionActive(ctx, namespace)
},
)
}
func (r *NamespaceReconciler) ensureFinalDeletionActive(
ctx context.Context,
namespace string,
) error {
current := &corev1.Namespace{}
if err := r.Client.Get(ctx, client.ObjectKey{Name: namespace}, current); err != nil {
if errors.IsNotFound(err) {
return fmt.Errorf(
"%w: namespace %s no longer exists",
errFinalDeletionCancelled,
namespace,
)
}
return fmt.Errorf(
"failed to verify final deletion status for namespace %s: %w",
namespace,
err,
)
}
if current.Status.Phase == corev1.NamespaceTerminating {
return fmt.Errorf(
"%w: namespace %s is terminating",
errFinalDeletionCancelled,
namespace,
)
}
if current.Annotations[types.DebtNamespaceAnnoStatusKey] != types.FinalDeletionDebtNamespaceAnnoStatus {
return fmt.Errorf(
"%w: namespace %s has debt status %q",
errFinalDeletionCancelled,
namespace,
current.Annotations[types.DebtNamespaceAnnoStatusKey],
)
}
return nil
}
func (r *NamespaceReconciler) DeleteUserResource(ctx context.Context, namespace string) error {
if err := r.ensureFinalDeletionActive(ctx, namespace); err != nil {
return err
}
// Delete backup first and wait for completion
if err := r.deleteBackup(ctx, namespace); err != nil {
return err
@@ -472,11 +531,22 @@ func (r *NamespaceReconciler) DeleteUserResource(ctx context.Context, namespace
}
}(rs)
}
// Cancellation is expected when recharge changes the namespace status while
// deletion workers are still running. Record it and collect sibling results
// so this expected cancellation does not become a reconcile failure.
wasCancelled := false
for range deleteResources {
if err := <-errChan; err != nil {
if errors2.Is(err, errFinalDeletionCancelled) {
wasCancelled = true
continue
}
return err
}
}
if wasCancelled {
return errFinalDeletionCancelled
}
return nil
}
@@ -1370,11 +1440,15 @@ func (AnnotationChangedPredicate) Update(e event.UpdateEvent) bool {
newDebtStatus := newObj.Annotations[types.DebtNamespaceAnnoStatusKey]
oldNetworkStatus := oldObj.Annotations[types.NetworkStatusAnnoKey]
newNetworkStatus := newObj.Annotations[types.NetworkStatusAnnoKey]
oldFinalDeletionReplay := oldObj.Annotations[types.FinalDeletionReplayAnnotationKey]
newFinalDeletionReplay := newObj.Annotations[types.FinalDeletionReplayAnnotationKey]
debtChanged := oldDebtStatus != newDebtStatus && !isDebtCompleted(newDebtStatus)
networkChanged := oldNetworkStatus != newNetworkStatus && !isNetworkCompleted(newNetworkStatus)
replayChanged := oldFinalDeletionReplay != newFinalDeletionReplay &&
newFinalDeletionReplay != ""
return debtChanged || networkChanged
return debtChanged || networkChanged || replayChanged
}
func (AnnotationChangedPredicate) Create(e event.CreateEvent) bool {
@@ -2370,6 +2444,10 @@ func (r *NamespaceReconciler) deleteResource(
ctx context.Context,
resource, namespace string,
) error {
if err := r.ensureFinalDeletionActive(ctx, namespace); err != nil {
return err
}
deletePolicy := v12.DeletePropagationForeground
var gvr schema.GroupVersionResource
switch resource {
@@ -2803,7 +2881,12 @@ func deleteResourceListAndWait(
gvr schema.GroupVersionResource,
namespace string,
semaphore chan struct{},
guard func(context.Context) error,
) error {
if err := guard(ctx); err != nil {
return err
}
// List all resources
list, err := dynamicClient.Resource(gvr).Namespace(namespace).List(ctx, v12.ListOptions{})
if err != nil {
@@ -2828,6 +2911,10 @@ func deleteResourceListAndWait(
select {
case semaphore <- struct{}{}:
defer func() { <-semaphore }() // Release semaphore when done
if guardErr := guard(ctx); guardErr != nil {
errCh <- guardErr
return
}
if deleteErr := deleteResourceAndWait(
ctx,
dynamicClient,
@@ -2849,13 +2936,24 @@ func deleteResourceListAndWait(
close(errCh)
}()
// Cancellation is an expected result when recharge changes the namespace
// status during cleanup. It must be separated from real deletion errors so
// the caller can stop cleanly after every worker has finished.
wasCancelled := false
for deleteErr := range errCh {
if errors2.Is(deleteErr, errFinalDeletionCancelled) {
wasCancelled = true
continue
}
allErrors = append(allErrors, deleteErr)
}
if len(allErrors) > 0 {
return fmt.Errorf("failed to delete some %s resources: %v", gvr, allErrors)
}
if wasCancelled {
return errFinalDeletionCancelled
}
return nil
}
@@ -3,9 +3,11 @@ package controllers
import (
"context"
"encoding/json"
"errors"
"testing"
"github.com/go-logr/logr"
"github.com/labring/sealos/controllers/pkg/types"
appsv1 "k8s.io/api/apps/v1"
autoscalingv2 "k8s.io/api/autoscaling/v2"
batchv1 "k8s.io/api/batch/v1"
@@ -18,8 +20,77 @@ import (
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/event"
)
func TestAnnotationChangedPredicateFinalDeletionReplay(t *testing.T) {
oldNamespace := &corev1.Namespace{
ObjectMeta: v1.ObjectMeta{
Annotations: map[string]string{
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionDebtNamespaceAnnoStatus,
},
},
}
newNamespace := oldNamespace.DeepCopy()
newNamespace.Annotations[types.FinalDeletionReplayAnnotationKey] = "2026-08-17T00:00:00Z"
if !(AnnotationChangedPredicate{}).Update(event.UpdateEvent{
ObjectOld: oldNamespace,
ObjectNew: newNamespace,
}) {
t.Fatal("final deletion replay annotation should trigger reconciliation")
}
oldNamespace = newNamespace
newNamespace = oldNamespace.DeepCopy()
newNamespace.Annotations[types.DebtNamespaceAnnoStatusKey] = types.ResumeCompletedDebtNamespaceAnnoStatus
if (AnnotationChangedPredicate{}).Update(event.UpdateEvent{
ObjectOld: oldNamespace,
ObjectNew: newNamespace,
}) {
t.Fatal("completed resume transition should not trigger deletion reconciliation")
}
}
func TestDeleteUserResourceStopsAfterRecharge(t *testing.T) {
scheme := runtime.NewScheme()
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatalf("failed to add core scheme: %v", err)
}
namespace := &corev1.Namespace{
ObjectMeta: v1.ObjectMeta{
Name: "test-ns",
Annotations: map[string]string{
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionDebtNamespaceAnnoStatus,
},
},
}
fakeClient := clientfake.NewClientBuilder().
WithScheme(scheme).
WithObjects(namespace).
Build()
ctx := context.Background()
var recharged corev1.Namespace
if err := fakeClient.Get(ctx, client.ObjectKey{Name: "test-ns"}, &recharged); err != nil {
t.Fatalf("failed to get namespace: %v", err)
}
recharged.Annotations[types.DebtNamespaceAnnoStatusKey] = types.ResumeDebtNamespaceAnnoStatus
if err := fakeClient.Update(ctx, &recharged); err != nil {
t.Fatalf("failed to update namespace after recharge: %v", err)
}
reconciler := &NamespaceReconciler{
Client: fakeClient,
Log: logr.Discard(),
}
err := reconciler.DeleteUserResource(ctx, "test-ns")
if !errors.Is(err, errFinalDeletionCancelled) {
t.Fatalf("expected final deletion cancellation, got %v", err)
}
}
// Test suspendOrphanDeployments and resumeOrphanDeployments
func TestSuspendResumeOrphanDeployments(t *testing.T) {
tests := []struct {
@@ -224,6 +224,18 @@ func (r *PaymentReconciler) reconcilePayment(payment *accountv1.Payment) error {
}
r.userLock[userUID].Lock()
defer r.userLock[userUID].Unlock()
debtMutex, err := r.DebtReconciler.getUserMutex(userUID)
if err != nil {
return fmt.Errorf("get debt user lock failed: %w", err)
}
debtMutex.Lock()
shouldRefreshDebt := false
defer func() {
debtMutex.Unlock()
if shouldRefreshDebt {
go r.DebtReconciler.processUsersInParallel([]uuid.UUID{userUID})
}
}()
userDiscount, err := r.Account.AccountV2.GetUserRechargeDiscount(
&pkgtypes.UserQueryOpts{ID: payment.Spec.UserID},
)
@@ -253,12 +265,12 @@ func (r *PaymentReconciler) reconcilePayment(payment *accountv1.Payment) error {
}); err != nil {
return fmt.Errorf("payment failed: %w", err)
}
shouldRefreshDebt = true
payment.Status.Status = pay.PaymentSuccess
if err := r.Status().Update(context.Background(), payment); err != nil {
return fmt.Errorf("update payment failed: %w", err)
}
go r.DebtReconciler.processUsersInParallel([]uuid.UUID{userUID})
// case pay.PaymentFailed, pay.PaymentExpired:
// case pay.PaymentFailed, pay.PaymentExpired:
default:
if err := r.expiredOvertimePayment(payment); err != nil {
return fmt.Errorf("expired payment failed: %w", err)
+1
View File
@@ -13,6 +13,7 @@ const (
SuspendCompletedDebtNamespaceAnnoStatus = "SuspendCompleted"
FinalDeletionDebtNamespaceAnnoStatus = "FinalDeletion"
FinalDeletionCompletedDebtNamespaceAnnoStatus = "FinalDeletionCompleted"
FinalDeletionReplayAnnotationKey = "debt.sealos/final-deletion-replay"
ResumeDebtNamespaceAnnoStatus = "Resume"
ResumeCompletedDebtNamespaceAnnoStatus = "ResumeCompleted"
TerminateSuspendDebtNamespaceAnnoStatus = "TerminateSuspend"
+63
View File
@@ -75,12 +75,75 @@ func adminFlushDebtResourceStatus(req *helper.AdminFlushDebtResourceStatusReq) e
if err != nil {
return fmt.Errorf("get own namespace list failed: %w", err)
}
if req.ReplayFinalDeletion {
if req.CurrentDebtStatus != types.FinalDeletionPeriod {
return fmt.Errorf(
"final deletion replay requires current status %s, got %q",
types.FinalDeletionPeriod,
req.CurrentDebtStatus,
)
}
if err = replayFinalDeletionNamespaceStatus(
context.Background(),
dao.K8sManager.GetClient(),
namespaces,
); err != nil {
return fmt.Errorf("failed to replay final deletion namespace status: %w", err)
}
return nil
}
if err = flushUserDebtResourceStatus(req, dao.K8sManager.GetClient(), namespaces); err != nil {
return fmt.Errorf("failed to flush user resource status: %w", err)
}
return nil
}
func replayFinalDeletionNamespaceStatus(
ctx context.Context,
clt client.Client,
namespaces []string,
) error {
for _, namespace := range namespaces {
original := &corev1.Namespace{}
if err := clt.Get(ctx, types2.NamespacedName{Name: namespace}, original); err != nil {
return err
}
if original.Status.Phase == corev1.NamespaceTerminating {
continue
}
currentStatus := original.Annotations[types.DebtNamespaceAnnoStatusKey]
switch currentStatus {
case types.ResumeDebtNamespaceAnnoStatus,
types.ResumeCompletedDebtNamespaceAnnoStatus,
types.FinalDeletionCompletedDebtNamespaceAnnoStatus:
// A recharge has already won this namespace transition. A stale
// replay must never turn it back into FinalDeletion. Completed
// namespaces are already reconciled and need no retry marker.
continue
}
ns := original.DeepCopy()
if ns.Annotations == nil {
ns.Annotations = make(map[string]string)
}
switch currentStatus {
case types.FinalDeletionDebtNamespaceAnnoStatus:
// Changing a separate annotation creates a new namespace event while
// keeping the destructive state explicit and idempotent.
ns.Annotations[types.FinalDeletionReplayAnnotationKey] = time.Now().
UTC().
Format(time.RFC3339Nano)
default:
ns.Annotations[types.DebtNamespaceAnnoStatusKey] = types.FinalDeletionDebtNamespaceAnnoStatus
}
if err := clt.Patch(ctx, ns, client.MergeFrom(original)); err != nil {
return fmt.Errorf("patch namespace %s for final deletion replay: %w", namespace, err)
}
}
return nil
}
func isDebtRecoveryTransition(lastStatus, currentStatus types.DebtStatusType) bool {
return types.ContainDebtStatus(types.DebtStates, lastStatus) &&
types.ContainDebtStatus(types.NonDebtStates, currentStatus)
+53
View File
@@ -147,6 +147,59 @@ func TestIsDebtRecoveryTransition(t *testing.T) {
}
}
func TestReplayFinalDeletionNamespaceStatusProtectsResumedNamespaces(t *testing.T) {
scheme := runtime.NewScheme()
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatalf("add core scheme: %v", err)
}
clt := clientfake.NewClientBuilder().
WithScheme(scheme).
WithObjects(
newTestNamespace("resumed", "owner", map[string]string{
types.DebtNamespaceAnnoStatusKey: types.ResumeCompletedDebtNamespaceAnnoStatus,
}),
newTestNamespace("in-progress", "owner", map[string]string{
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionDebtNamespaceAnnoStatus,
}),
newTestNamespace("completed", "owner", map[string]string{
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionCompletedDebtNamespaceAnnoStatus,
}),
newTestNamespace("missing", "owner", nil),
).
Build()
if err := replayFinalDeletionNamespaceStatus(
context.Background(),
clt,
[]string{"resumed", "in-progress", "completed", "missing"},
); err != nil {
t.Fatalf("replay final deletion namespace status: %v", err)
}
assertNamespaceDebtStatus := func(name, want string) *corev1.Namespace {
t.Helper()
ns := &corev1.Namespace{}
if err := clt.Get(context.Background(), client.ObjectKey{Name: name}, ns); err != nil {
t.Fatalf("get namespace %s: %v", name, err)
}
if got := ns.Annotations[types.DebtNamespaceAnnoStatusKey]; got != want {
t.Fatalf("namespace %s debt status = %q, want %q", name, got, want)
}
return ns
}
assertNamespaceDebtStatus("resumed", types.ResumeCompletedDebtNamespaceAnnoStatus)
if ns := assertNamespaceDebtStatus(
"in-progress",
types.FinalDeletionDebtNamespaceAnnoStatus,
); ns.Annotations[types.FinalDeletionReplayAnnotationKey] == "" {
t.Fatal("in-progress namespace should receive a replay marker")
}
assertNamespaceDebtStatus("completed", types.FinalDeletionCompletedDebtNamespaceAnnoStatus)
assertNamespaceDebtStatus("missing", types.FinalDeletionDebtNamespaceAnnoStatus)
}
func setWorkspaceSubscriptionExistsForTest(t *testing.T, fn func(workspace string) (bool, error)) {
t.Helper()
old := workspaceSubscriptionExists
+5 -4
View File
@@ -675,10 +675,11 @@ func ParseAdminFlushSubscriptionQuotaReq(c *gin.Context) (*AdminFlushSubscriptio
}
type AdminFlushDebtResourceStatusReq struct {
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
ReplayFinalDeletion bool `json:"replayFinalDeletion,omitempty" bson:"replayFinalDeletion,omitempty"`
}
func ParseAdminFlushDebtResourceStatusReq(