mirror of
https://github.com/labring/sealos.git
synced 2026-08-29 01:39:49 +08:00
fix(account): prevent stale debt deletion after recharge (#7230)
* fix(account): prevent stale debt deletion after recharge * style(account): satisfy golangci-lint formatting * fix(account): improve debt deletion diagnostics
This commit is contained in:
@@ -720,9 +720,50 @@ func (r *DebtReconciler) syncFinalDeletionDebtNamespacesForUser(
|
||||
ctx context.Context,
|
||||
userUID uuid.UUID,
|
||||
) error {
|
||||
mutex, err := r.getUserMutex(userUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
// The hourly query can race with a successful recharge. Recompute the
|
||||
// current debt state while holding the same lock as normal refreshes before
|
||||
// issuing another destructive cleanup request.
|
||||
var debt types.Debt
|
||||
if err := r.AccountV2.GetGlobalDB().
|
||||
Where("user_uid = ?", userUID).
|
||||
First(&debt).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to reload debt %s: %w", userUID, err)
|
||||
}
|
||||
if debt.AccountDebtStatus != types.FinalDeletionPeriod {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := r.refreshDebtStatus(userUID, true); err != nil {
|
||||
return fmt.Errorf("failed to refresh debt status before final deletion replay: %w", err)
|
||||
}
|
||||
|
||||
if err := r.AccountV2.GetGlobalDB().
|
||||
Where("user_uid = ?", userUID).
|
||||
First(&debt).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to reload debt %s after refresh: %w", userUID, err)
|
||||
}
|
||||
if debt.AccountDebtStatus != types.FinalDeletionPeriod {
|
||||
return nil
|
||||
}
|
||||
|
||||
req := finalDeletionDebtNamespaceFlushReq(userUID)
|
||||
req.ReplayFinalDeletion = true
|
||||
return r.sendFlushDebtResourceStatusRequestWithContext(
|
||||
ctx,
|
||||
finalDeletionDebtNamespaceFlushReq(userUID),
|
||||
req,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1055,10 +1096,11 @@ func (r *DebtReconciler) SendUserDebtMsg(
|
||||
}
|
||||
|
||||
type AdminFlushResourceStatusReq struct {
|
||||
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
|
||||
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
|
||||
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
|
||||
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
|
||||
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
|
||||
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
|
||||
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
|
||||
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
|
||||
ReplayFinalDeletion bool `json:"replayFinalDeletion,omitempty" bson:"replayFinalDeletion,omitempty"`
|
||||
}
|
||||
|
||||
// TODO flush desktop message (send or read) && flush resource quota (suspend or resume or delete)
|
||||
@@ -1213,6 +1255,18 @@ func (r *DebtReconciler) retryFailedUsers() {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *DebtReconciler) getUserMutex(userUID uuid.UUID) (*sync.Mutex, error) {
|
||||
if r.userLocks == nil {
|
||||
return nil, errors.New("user locks are not initialized")
|
||||
}
|
||||
lock, _ := r.userLocks.LoadOrStore(userUID, &sync.Mutex{})
|
||||
mutex, ok := lock.(*sync.Mutex)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid mutex for user %s", userUID)
|
||||
}
|
||||
return mutex, nil
|
||||
}
|
||||
|
||||
// Parallel processing of user debt status, the same user simultaneously through the lock to implement a debt refresh processing.
|
||||
func (r *DebtReconciler) processUsersInParallel(users []uuid.UUID) {
|
||||
var (
|
||||
@@ -1226,13 +1280,9 @@ func (r *DebtReconciler) processUsersInParallel(users []uuid.UUID) {
|
||||
go func(u uuid.UUID) {
|
||||
defer wg.Done()
|
||||
defer func() { <-semaphore }()
|
||||
lock, _ := r.userLocks.LoadOrStore(u, &sync.Mutex{})
|
||||
mutex, ok := lock.(*sync.Mutex)
|
||||
if !ok {
|
||||
r.Error(
|
||||
fmt.Errorf("invalid mutex for user %s", u),
|
||||
"failed to load user mutex",
|
||||
)
|
||||
mutex, err := r.getUserMutex(u)
|
||||
if err != nil {
|
||||
r.Error(err, "failed to load user mutex", "userUID", u)
|
||||
return
|
||||
}
|
||||
if !mutex.TryLock() {
|
||||
|
||||
@@ -29,4 +29,17 @@ func TestFinalDeletionDebtNamespaceFlushReq(t *testing.T) {
|
||||
req.CurrentDebtStatus,
|
||||
)
|
||||
}
|
||||
if req.ReplayFinalDeletion {
|
||||
t.Fatal("normal final deletion request should not be marked as replay")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalDeletionDebtNamespaceReplayReq(t *testing.T) {
|
||||
userUID := uuid.New()
|
||||
req := finalDeletionDebtNamespaceFlushReq(userUID)
|
||||
req.ReplayFinalDeletion = true
|
||||
|
||||
if !req.ReplayFinalDeletion {
|
||||
t.Fatal("final deletion replay request should be marked as replay")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,6 +74,8 @@ const (
|
||||
DeployPVCResizeKey = "deploy.cloud.sealos.io/resize"
|
||||
)
|
||||
|
||||
var errFinalDeletionCancelled = errors2.New("final deletion cancelled by namespace status")
|
||||
|
||||
//+kubebuilder:rbac:groups=core,resources=namespaces,verbs=get;list;watch;create;update;patch;delete
|
||||
//+kubebuilder:rbac:groups=core,resources=namespaces/status,verbs=get;update;patch
|
||||
//+kubebuilder:rbac:groups=core,resources=namespaces/finalizers,verbs=update
|
||||
@@ -146,6 +148,9 @@ func (r *NamespaceReconciler) Reconcile(
|
||||
// auxiliary function handles resource operations
|
||||
performAction := func(action func(context.Context, string) error, actionName string) (ctrl.Result, error) {
|
||||
if err := action(ctx, req.Name); err != nil {
|
||||
if errors2.Is(err, errFinalDeletionCancelled) {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
logger.Error(err, actionName+" namespace resources failed")
|
||||
return ctrl.Result{
|
||||
Requeue: actionName == deleteConst,
|
||||
@@ -379,6 +384,10 @@ func (r *NamespaceReconciler) Reconcile(
|
||||
if t.condition() {
|
||||
if t.action != nil {
|
||||
if result, err := performAction(t.action, t.actionName); err != nil {
|
||||
if errors2.Is(err, errFinalDeletionCancelled) {
|
||||
logger.Info("final deletion cancelled by current namespace status")
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
@@ -441,10 +450,60 @@ func (r *NamespaceReconciler) deleteBackup(ctx context.Context, namespace string
|
||||
Version: "v1alpha1",
|
||||
Resource: "backups",
|
||||
}
|
||||
return deleteResourceListAndWait(ctx, r.dynamicClient, gvr, namespace, r.deleteBackupSemaphore)
|
||||
return deleteResourceListAndWait(
|
||||
ctx,
|
||||
r.dynamicClient,
|
||||
gvr,
|
||||
namespace,
|
||||
r.deleteBackupSemaphore,
|
||||
func(ctx context.Context) error {
|
||||
return r.ensureFinalDeletionActive(ctx, namespace)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (r *NamespaceReconciler) ensureFinalDeletionActive(
|
||||
ctx context.Context,
|
||||
namespace string,
|
||||
) error {
|
||||
current := &corev1.Namespace{}
|
||||
if err := r.Client.Get(ctx, client.ObjectKey{Name: namespace}, current); err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
return fmt.Errorf(
|
||||
"%w: namespace %s no longer exists",
|
||||
errFinalDeletionCancelled,
|
||||
namespace,
|
||||
)
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"failed to verify final deletion status for namespace %s: %w",
|
||||
namespace,
|
||||
err,
|
||||
)
|
||||
}
|
||||
if current.Status.Phase == corev1.NamespaceTerminating {
|
||||
return fmt.Errorf(
|
||||
"%w: namespace %s is terminating",
|
||||
errFinalDeletionCancelled,
|
||||
namespace,
|
||||
)
|
||||
}
|
||||
if current.Annotations[types.DebtNamespaceAnnoStatusKey] != types.FinalDeletionDebtNamespaceAnnoStatus {
|
||||
return fmt.Errorf(
|
||||
"%w: namespace %s has debt status %q",
|
||||
errFinalDeletionCancelled,
|
||||
namespace,
|
||||
current.Annotations[types.DebtNamespaceAnnoStatusKey],
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *NamespaceReconciler) DeleteUserResource(ctx context.Context, namespace string) error {
|
||||
if err := r.ensureFinalDeletionActive(ctx, namespace); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete backup first and wait for completion
|
||||
if err := r.deleteBackup(ctx, namespace); err != nil {
|
||||
return err
|
||||
@@ -472,11 +531,22 @@ func (r *NamespaceReconciler) DeleteUserResource(ctx context.Context, namespace
|
||||
}
|
||||
}(rs)
|
||||
}
|
||||
// Cancellation is expected when recharge changes the namespace status while
|
||||
// deletion workers are still running. Record it and collect sibling results
|
||||
// so this expected cancellation does not become a reconcile failure.
|
||||
wasCancelled := false
|
||||
for range deleteResources {
|
||||
if err := <-errChan; err != nil {
|
||||
if errors2.Is(err, errFinalDeletionCancelled) {
|
||||
wasCancelled = true
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
if wasCancelled {
|
||||
return errFinalDeletionCancelled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1370,11 +1440,15 @@ func (AnnotationChangedPredicate) Update(e event.UpdateEvent) bool {
|
||||
newDebtStatus := newObj.Annotations[types.DebtNamespaceAnnoStatusKey]
|
||||
oldNetworkStatus := oldObj.Annotations[types.NetworkStatusAnnoKey]
|
||||
newNetworkStatus := newObj.Annotations[types.NetworkStatusAnnoKey]
|
||||
oldFinalDeletionReplay := oldObj.Annotations[types.FinalDeletionReplayAnnotationKey]
|
||||
newFinalDeletionReplay := newObj.Annotations[types.FinalDeletionReplayAnnotationKey]
|
||||
|
||||
debtChanged := oldDebtStatus != newDebtStatus && !isDebtCompleted(newDebtStatus)
|
||||
networkChanged := oldNetworkStatus != newNetworkStatus && !isNetworkCompleted(newNetworkStatus)
|
||||
replayChanged := oldFinalDeletionReplay != newFinalDeletionReplay &&
|
||||
newFinalDeletionReplay != ""
|
||||
|
||||
return debtChanged || networkChanged
|
||||
return debtChanged || networkChanged || replayChanged
|
||||
}
|
||||
|
||||
func (AnnotationChangedPredicate) Create(e event.CreateEvent) bool {
|
||||
@@ -2370,6 +2444,10 @@ func (r *NamespaceReconciler) deleteResource(
|
||||
ctx context.Context,
|
||||
resource, namespace string,
|
||||
) error {
|
||||
if err := r.ensureFinalDeletionActive(ctx, namespace); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
deletePolicy := v12.DeletePropagationForeground
|
||||
var gvr schema.GroupVersionResource
|
||||
switch resource {
|
||||
@@ -2803,7 +2881,12 @@ func deleteResourceListAndWait(
|
||||
gvr schema.GroupVersionResource,
|
||||
namespace string,
|
||||
semaphore chan struct{},
|
||||
guard func(context.Context) error,
|
||||
) error {
|
||||
if err := guard(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// List all resources
|
||||
list, err := dynamicClient.Resource(gvr).Namespace(namespace).List(ctx, v12.ListOptions{})
|
||||
if err != nil {
|
||||
@@ -2828,6 +2911,10 @@ func deleteResourceListAndWait(
|
||||
select {
|
||||
case semaphore <- struct{}{}:
|
||||
defer func() { <-semaphore }() // Release semaphore when done
|
||||
if guardErr := guard(ctx); guardErr != nil {
|
||||
errCh <- guardErr
|
||||
return
|
||||
}
|
||||
if deleteErr := deleteResourceAndWait(
|
||||
ctx,
|
||||
dynamicClient,
|
||||
@@ -2849,13 +2936,24 @@ func deleteResourceListAndWait(
|
||||
close(errCh)
|
||||
}()
|
||||
|
||||
// Cancellation is an expected result when recharge changes the namespace
|
||||
// status during cleanup. It must be separated from real deletion errors so
|
||||
// the caller can stop cleanly after every worker has finished.
|
||||
wasCancelled := false
|
||||
for deleteErr := range errCh {
|
||||
if errors2.Is(deleteErr, errFinalDeletionCancelled) {
|
||||
wasCancelled = true
|
||||
continue
|
||||
}
|
||||
allErrors = append(allErrors, deleteErr)
|
||||
}
|
||||
|
||||
if len(allErrors) > 0 {
|
||||
return fmt.Errorf("failed to delete some %s resources: %v", gvr, allErrors)
|
||||
}
|
||||
if wasCancelled {
|
||||
return errFinalDeletionCancelled
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,9 +3,11 @@ package controllers
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/labring/sealos/controllers/pkg/types"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
autoscalingv2 "k8s.io/api/autoscaling/v2"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
@@ -18,8 +20,77 @@ import (
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
)
|
||||
|
||||
func TestAnnotationChangedPredicateFinalDeletionReplay(t *testing.T) {
|
||||
oldNamespace := &corev1.Namespace{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Annotations: map[string]string{
|
||||
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionDebtNamespaceAnnoStatus,
|
||||
},
|
||||
},
|
||||
}
|
||||
newNamespace := oldNamespace.DeepCopy()
|
||||
newNamespace.Annotations[types.FinalDeletionReplayAnnotationKey] = "2026-08-17T00:00:00Z"
|
||||
|
||||
if !(AnnotationChangedPredicate{}).Update(event.UpdateEvent{
|
||||
ObjectOld: oldNamespace,
|
||||
ObjectNew: newNamespace,
|
||||
}) {
|
||||
t.Fatal("final deletion replay annotation should trigger reconciliation")
|
||||
}
|
||||
|
||||
oldNamespace = newNamespace
|
||||
newNamespace = oldNamespace.DeepCopy()
|
||||
newNamespace.Annotations[types.DebtNamespaceAnnoStatusKey] = types.ResumeCompletedDebtNamespaceAnnoStatus
|
||||
if (AnnotationChangedPredicate{}).Update(event.UpdateEvent{
|
||||
ObjectOld: oldNamespace,
|
||||
ObjectNew: newNamespace,
|
||||
}) {
|
||||
t.Fatal("completed resume transition should not trigger deletion reconciliation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteUserResourceStopsAfterRecharge(t *testing.T) {
|
||||
scheme := runtime.NewScheme()
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
t.Fatalf("failed to add core scheme: %v", err)
|
||||
}
|
||||
|
||||
namespace := &corev1.Namespace{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: "test-ns",
|
||||
Annotations: map[string]string{
|
||||
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionDebtNamespaceAnnoStatus,
|
||||
},
|
||||
},
|
||||
}
|
||||
fakeClient := clientfake.NewClientBuilder().
|
||||
WithScheme(scheme).
|
||||
WithObjects(namespace).
|
||||
Build()
|
||||
|
||||
ctx := context.Background()
|
||||
var recharged corev1.Namespace
|
||||
if err := fakeClient.Get(ctx, client.ObjectKey{Name: "test-ns"}, &recharged); err != nil {
|
||||
t.Fatalf("failed to get namespace: %v", err)
|
||||
}
|
||||
recharged.Annotations[types.DebtNamespaceAnnoStatusKey] = types.ResumeDebtNamespaceAnnoStatus
|
||||
if err := fakeClient.Update(ctx, &recharged); err != nil {
|
||||
t.Fatalf("failed to update namespace after recharge: %v", err)
|
||||
}
|
||||
|
||||
reconciler := &NamespaceReconciler{
|
||||
Client: fakeClient,
|
||||
Log: logr.Discard(),
|
||||
}
|
||||
err := reconciler.DeleteUserResource(ctx, "test-ns")
|
||||
if !errors.Is(err, errFinalDeletionCancelled) {
|
||||
t.Fatalf("expected final deletion cancellation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test suspendOrphanDeployments and resumeOrphanDeployments
|
||||
func TestSuspendResumeOrphanDeployments(t *testing.T) {
|
||||
tests := []struct {
|
||||
|
||||
@@ -224,6 +224,18 @@ func (r *PaymentReconciler) reconcilePayment(payment *accountv1.Payment) error {
|
||||
}
|
||||
r.userLock[userUID].Lock()
|
||||
defer r.userLock[userUID].Unlock()
|
||||
debtMutex, err := r.DebtReconciler.getUserMutex(userUID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get debt user lock failed: %w", err)
|
||||
}
|
||||
debtMutex.Lock()
|
||||
shouldRefreshDebt := false
|
||||
defer func() {
|
||||
debtMutex.Unlock()
|
||||
if shouldRefreshDebt {
|
||||
go r.DebtReconciler.processUsersInParallel([]uuid.UUID{userUID})
|
||||
}
|
||||
}()
|
||||
userDiscount, err := r.Account.AccountV2.GetUserRechargeDiscount(
|
||||
&pkgtypes.UserQueryOpts{ID: payment.Spec.UserID},
|
||||
)
|
||||
@@ -253,12 +265,12 @@ func (r *PaymentReconciler) reconcilePayment(payment *accountv1.Payment) error {
|
||||
}); err != nil {
|
||||
return fmt.Errorf("payment failed: %w", err)
|
||||
}
|
||||
shouldRefreshDebt = true
|
||||
payment.Status.Status = pay.PaymentSuccess
|
||||
if err := r.Status().Update(context.Background(), payment); err != nil {
|
||||
return fmt.Errorf("update payment failed: %w", err)
|
||||
}
|
||||
go r.DebtReconciler.processUsersInParallel([]uuid.UUID{userUID})
|
||||
// case pay.PaymentFailed, pay.PaymentExpired:
|
||||
// case pay.PaymentFailed, pay.PaymentExpired:
|
||||
default:
|
||||
if err := r.expiredOvertimePayment(payment); err != nil {
|
||||
return fmt.Errorf("expired payment failed: %w", err)
|
||||
|
||||
@@ -13,6 +13,7 @@ const (
|
||||
SuspendCompletedDebtNamespaceAnnoStatus = "SuspendCompleted"
|
||||
FinalDeletionDebtNamespaceAnnoStatus = "FinalDeletion"
|
||||
FinalDeletionCompletedDebtNamespaceAnnoStatus = "FinalDeletionCompleted"
|
||||
FinalDeletionReplayAnnotationKey = "debt.sealos/final-deletion-replay"
|
||||
ResumeDebtNamespaceAnnoStatus = "Resume"
|
||||
ResumeCompletedDebtNamespaceAnnoStatus = "ResumeCompleted"
|
||||
TerminateSuspendDebtNamespaceAnnoStatus = "TerminateSuspend"
|
||||
|
||||
@@ -75,12 +75,75 @@ func adminFlushDebtResourceStatus(req *helper.AdminFlushDebtResourceStatusReq) e
|
||||
if err != nil {
|
||||
return fmt.Errorf("get own namespace list failed: %w", err)
|
||||
}
|
||||
if req.ReplayFinalDeletion {
|
||||
if req.CurrentDebtStatus != types.FinalDeletionPeriod {
|
||||
return fmt.Errorf(
|
||||
"final deletion replay requires current status %s, got %q",
|
||||
types.FinalDeletionPeriod,
|
||||
req.CurrentDebtStatus,
|
||||
)
|
||||
}
|
||||
if err = replayFinalDeletionNamespaceStatus(
|
||||
context.Background(),
|
||||
dao.K8sManager.GetClient(),
|
||||
namespaces,
|
||||
); err != nil {
|
||||
return fmt.Errorf("failed to replay final deletion namespace status: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err = flushUserDebtResourceStatus(req, dao.K8sManager.GetClient(), namespaces); err != nil {
|
||||
return fmt.Errorf("failed to flush user resource status: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func replayFinalDeletionNamespaceStatus(
|
||||
ctx context.Context,
|
||||
clt client.Client,
|
||||
namespaces []string,
|
||||
) error {
|
||||
for _, namespace := range namespaces {
|
||||
original := &corev1.Namespace{}
|
||||
if err := clt.Get(ctx, types2.NamespacedName{Name: namespace}, original); err != nil {
|
||||
return err
|
||||
}
|
||||
if original.Status.Phase == corev1.NamespaceTerminating {
|
||||
continue
|
||||
}
|
||||
|
||||
currentStatus := original.Annotations[types.DebtNamespaceAnnoStatusKey]
|
||||
switch currentStatus {
|
||||
case types.ResumeDebtNamespaceAnnoStatus,
|
||||
types.ResumeCompletedDebtNamespaceAnnoStatus,
|
||||
types.FinalDeletionCompletedDebtNamespaceAnnoStatus:
|
||||
// A recharge has already won this namespace transition. A stale
|
||||
// replay must never turn it back into FinalDeletion. Completed
|
||||
// namespaces are already reconciled and need no retry marker.
|
||||
continue
|
||||
}
|
||||
|
||||
ns := original.DeepCopy()
|
||||
if ns.Annotations == nil {
|
||||
ns.Annotations = make(map[string]string)
|
||||
}
|
||||
switch currentStatus {
|
||||
case types.FinalDeletionDebtNamespaceAnnoStatus:
|
||||
// Changing a separate annotation creates a new namespace event while
|
||||
// keeping the destructive state explicit and idempotent.
|
||||
ns.Annotations[types.FinalDeletionReplayAnnotationKey] = time.Now().
|
||||
UTC().
|
||||
Format(time.RFC3339Nano)
|
||||
default:
|
||||
ns.Annotations[types.DebtNamespaceAnnoStatusKey] = types.FinalDeletionDebtNamespaceAnnoStatus
|
||||
}
|
||||
if err := clt.Patch(ctx, ns, client.MergeFrom(original)); err != nil {
|
||||
return fmt.Errorf("patch namespace %s for final deletion replay: %w", namespace, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isDebtRecoveryTransition(lastStatus, currentStatus types.DebtStatusType) bool {
|
||||
return types.ContainDebtStatus(types.DebtStates, lastStatus) &&
|
||||
types.ContainDebtStatus(types.NonDebtStates, currentStatus)
|
||||
|
||||
@@ -147,6 +147,59 @@ func TestIsDebtRecoveryTransition(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplayFinalDeletionNamespaceStatusProtectsResumedNamespaces(t *testing.T) {
|
||||
scheme := runtime.NewScheme()
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
t.Fatalf("add core scheme: %v", err)
|
||||
}
|
||||
|
||||
clt := clientfake.NewClientBuilder().
|
||||
WithScheme(scheme).
|
||||
WithObjects(
|
||||
newTestNamespace("resumed", "owner", map[string]string{
|
||||
types.DebtNamespaceAnnoStatusKey: types.ResumeCompletedDebtNamespaceAnnoStatus,
|
||||
}),
|
||||
newTestNamespace("in-progress", "owner", map[string]string{
|
||||
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionDebtNamespaceAnnoStatus,
|
||||
}),
|
||||
newTestNamespace("completed", "owner", map[string]string{
|
||||
types.DebtNamespaceAnnoStatusKey: types.FinalDeletionCompletedDebtNamespaceAnnoStatus,
|
||||
}),
|
||||
newTestNamespace("missing", "owner", nil),
|
||||
).
|
||||
Build()
|
||||
|
||||
if err := replayFinalDeletionNamespaceStatus(
|
||||
context.Background(),
|
||||
clt,
|
||||
[]string{"resumed", "in-progress", "completed", "missing"},
|
||||
); err != nil {
|
||||
t.Fatalf("replay final deletion namespace status: %v", err)
|
||||
}
|
||||
|
||||
assertNamespaceDebtStatus := func(name, want string) *corev1.Namespace {
|
||||
t.Helper()
|
||||
ns := &corev1.Namespace{}
|
||||
if err := clt.Get(context.Background(), client.ObjectKey{Name: name}, ns); err != nil {
|
||||
t.Fatalf("get namespace %s: %v", name, err)
|
||||
}
|
||||
if got := ns.Annotations[types.DebtNamespaceAnnoStatusKey]; got != want {
|
||||
t.Fatalf("namespace %s debt status = %q, want %q", name, got, want)
|
||||
}
|
||||
return ns
|
||||
}
|
||||
|
||||
assertNamespaceDebtStatus("resumed", types.ResumeCompletedDebtNamespaceAnnoStatus)
|
||||
if ns := assertNamespaceDebtStatus(
|
||||
"in-progress",
|
||||
types.FinalDeletionDebtNamespaceAnnoStatus,
|
||||
); ns.Annotations[types.FinalDeletionReplayAnnotationKey] == "" {
|
||||
t.Fatal("in-progress namespace should receive a replay marker")
|
||||
}
|
||||
assertNamespaceDebtStatus("completed", types.FinalDeletionCompletedDebtNamespaceAnnoStatus)
|
||||
assertNamespaceDebtStatus("missing", types.FinalDeletionDebtNamespaceAnnoStatus)
|
||||
}
|
||||
|
||||
func setWorkspaceSubscriptionExistsForTest(t *testing.T, fn func(workspace string) (bool, error)) {
|
||||
t.Helper()
|
||||
old := workspaceSubscriptionExists
|
||||
|
||||
@@ -675,10 +675,11 @@ func ParseAdminFlushSubscriptionQuotaReq(c *gin.Context) (*AdminFlushSubscriptio
|
||||
}
|
||||
|
||||
type AdminFlushDebtResourceStatusReq struct {
|
||||
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
|
||||
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
|
||||
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
|
||||
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
|
||||
UserUID uuid.UUID `json:"userUID" bson:"userUID"`
|
||||
LastDebtStatus types.DebtStatusType `json:"lastDebtStatus" bson:"lastDebtStatus"`
|
||||
CurrentDebtStatus types.DebtStatusType `json:"currentDebtStatus" bson:"currentDebtStatus"`
|
||||
IsBasicUser bool `json:"isBasicUser" bson:"isBasicUser"`
|
||||
ReplayFinalDeletion bool `json:"replayFinalDeletion,omitempty" bson:"replayFinalDeletion,omitempty"`
|
||||
}
|
||||
|
||||
func ParseAdminFlushDebtResourceStatusReq(
|
||||
|
||||
Reference in New Issue
Block a user