Merge pull request #5649 from swordqiu/automated-cherry-pick-of-#5648-upstream-release-3.1

Automated cherry pick of #5648: fix: cancel pending usage might not need to update usage

pkg/cloudcommon/db/db_dispatcher.go

@@ -1613,7 +1613,7 @@ func objectUpdateSpec(dispatcher *DBModelDispatcher, model IModel, modelValue re
 
 func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IModel) error {
 	// log.Debugf("Ready to delete %s %s %#v", jsonutils.Marshal(item), item, manager)
-	cleanModelUsages(ctx, userCred, item)
+	// cleanModelUsages(ctx, userCred, item)
 	_, err := Update(item, func() error {
 		return item.MarkDelete()
 	})

pkg/cloudcommon/db/quotas/context.go

@@ -74,7 +74,8 @@ func cancelPendingUsagesInContext(ctx context.Context, userCred mcclient.TokenCr
 	}
 	errs := make([]error, 0)
 	for i := range quotas {
-		err := CancelPendingUsage(ctx, userCred, quotas[i], quotas[i])
+		// cancel and do not save pending usage
+		err := CancelPendingUsage(ctx, userCred, quotas[i], quotas[i], false)
 		if err != nil {
 			errs = append(errs, errors.Wrapf(err, "CancelPendingUsage %s", jsonutils.Marshal(quotas[i])))
 		}

pkg/cloudcommon/db/quotas/interface.go

@@ -69,7 +69,7 @@ type IQuotaManager interface {
 	db.IResourceModelManager
 
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
-	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error
+	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
 	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 

pkg/cloudcommon/db/quotas/quotas.go

@@ -66,14 +66,14 @@ func (manager *SQuotaBaseManager) _cleanPendingUsage(ctx context.Context, userCr
 	return nil
 }
 
-func (manager *SQuotaBaseManager) cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func (manager *SQuotaBaseManager) cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	LockQuota(ctx, manager, localUsage)
 	defer ReleaseQuota(ctx, manager, localUsage)
 
-	return manager._cancelPendingUsage(ctx, userCred, localUsage, cancelUsage)
+	return manager._cancelPendingUsage(ctx, userCred, localUsage, cancelUsage, save)
 }
 
-func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	originKeys := localUsage.GetKeys()
 	// currentKeys := cancelUsage.GetKeys()
 
@@ -92,9 +92,11 @@ func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userC
 
 	log.Debugf("cancelUsage: %s localUsage: %s pendingUsage: %s", jsonutils.Marshal(cancelUsage), jsonutils.Marshal(localUsage), jsonutils.Marshal(pendingUsage))
 
-	err = manager.changeUsage(ctx, userCred, pendingUsage, true)
-	if err != nil {
-		return errors.Wrap(err, "manager.changelUsage")
+	if save {
+		err = manager.changeUsage(ctx, userCred, pendingUsage, true)
+		if err != nil {
+			return errors.Wrap(err, "manager.changelUsage")
+		}
 	}
 
 	return nil

pkg/cloudcommon/db/quotas/register.go

@@ -55,9 +55,9 @@ func getQuotaManager(quota IQuota) IQuotaManager {
 	}
 }
 
-func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	manager := getQuotaManager(cancelUsage)
-	return manager.cancelPendingUsage(ctx, userCred, localUsage, cancelUsage)
+	return manager.cancelPendingUsage(ctx, userCred, localUsage, cancelUsage, save)
 }
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {

pkg/compute/models/buckets.go

@@ -463,7 +463,7 @@ func (bucket *SBucket) PostCreate(
 		log.Errorf("bucket.GetQuotaKeys fail %s", err)
 	} else {
 		pendingUsage.SetKeys(keys)
-		err = quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+		err = quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
 		if err != nil {
 			log.Errorf("CancelPendingUsage error %s", err)
 		}
@@ -780,7 +780,7 @@ func (bucket *SBucket) PerformMakedir(
 
 	bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, true)
 
-	quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+	quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
 
 	return nil, nil
 }
@@ -940,12 +940,13 @@ func (bucket *SBucket) PerformUpload(
 			return nil, httperrors.NewOutOfQuotaError("%s", err)
 		}
 
-		// always cancel pending usage
-		defer quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
 	}
 
 	err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, cloudprovider.TBucketACLType(aclStr), storageClass, meta, false)
 	if err != nil {
+		if !pendingUsage.IsEmpty() {
+			quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		}
 		return nil, httperrors.NewInternalServerError("put object error %s", err)
 	}
 
@@ -954,6 +955,10 @@ func (bucket *SBucket) PerformUpload(
 
 	bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, true)
 
+	if !pendingUsage.IsEmpty() {
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
+	}
+
 	return nil, nil
 }
 

pkg/compute/models/elasticips.go

@@ -776,7 +776,7 @@ func (self *SElasticip) PostCreate(ctx context.Context, userCred mcclient.TokenC
 		log.Errorf("GetQuotaKeys fail %s", err)
 	} else {
 		eipPendingUsage.SetKeys(keys)
-		err := quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage)
+		err := quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage, true)
 		if err != nil {
 			log.Errorf("SElasticip CancelPendingUsage error: %s", err)
 		}
@@ -1140,7 +1140,7 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		host.GetCloudprovider(),
 	)
 	eipPendingUsage.SetKeys(keys)
-	quotas.CancelPendingUsage(ctx, userCred, pendingUsage, eipPendingUsage)
+	quotas.CancelPendingUsage(ctx, userCred, pendingUsage, eipPendingUsage, true)
 
 	return &eip, nil
 }

pkg/compute/models/guest_actions.go

@@ -1556,7 +1556,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok
 
 	err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage, false, false, nil, nil, false)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 		logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false)
 		return nil, httperrors.NewBadRequestError(err.Error())
 	}
@@ -2135,7 +2135,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.
 		_, err = self.attach2NetworkDesc(ctx, userCred, host, input.Nets[i], pendingUsage, nil)
 		logclient.AddSimpleActionLog(self, logclient.ACT_ATTACH_NETWORK, input.Nets[i], userCred, err == nil)
 		if err != nil {
-			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 			return nil, httperrors.NewBadRequestError(err.Error())
 		}
 	}
@@ -2146,7 +2146,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.
 		err = self.StartSyncTask(ctx, userCred, false, "")
 	}
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 	}
 	return nil, err
 }
@@ -2483,7 +2483,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T
 	if len(newDisks) > 0 {
 		err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage, false, false, nil, nil, false)
 		if err != nil {
-			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 			return nil, httperrors.NewBadRequestError("Create disk on host error: %s", err)
 		}
 		confs.Add(jsonutils.Marshal(newDisks), "create")
@@ -2968,7 +2968,7 @@ func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.Toke
 
 	eip, err := ElasticipManager.NewEipForVMOnHost(ctx, userCred, self, host, int(bw), chargeType, eipPendingUsage)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 
@@ -3363,7 +3363,7 @@ func (self *SGuest) PerformCreateBackup(ctx context.Context, userCred mcclient.T
 	params.Set("guest_status", jsonutils.NewString(self.Status))
 	task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateBackupTask", self, userCred, params, "", "", &req)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &req, &req)
+		quotas.CancelPendingUsage(ctx, userCred, &req, &req, false)
 		log.Errorln(err)
 		return nil, err
 	} else {
@@ -4318,13 +4318,13 @@ func (self *SGuest) PerformInstanceSnapshot(
 	instanceSnapshot, err := InstanceSnapshotManager.CreateInstanceSnapshot(ctx, userCred, self, name, false)
 	if err != nil {
 		quotas.CancelPendingUsage(
-			ctx, userCred, pendingUsage, pendingUsage)
+			ctx, userCred, pendingUsage, pendingUsage, false)
 		return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err)
 	}
 	err = self.InstaceCreateSnapshot(ctx, userCred, instanceSnapshot, pendingUsage)
 	if err != nil {
 		quotas.CancelPendingUsage(
-			ctx, userCred, pendingUsage, pendingUsage)
+			ctx, userCred, pendingUsage, pendingUsage, false)
 		return nil, httperrors.NewInternalServerError("start create snapshot task failed: %s", err)
 	}
 	return nil, nil
@@ -4422,49 +4422,52 @@ func (self *SGuest) PerformSnapshotAndClone(
 	pendingUsage, pendingRegionUsage, err := self.getGuestUsage(int(count))
 	keys, err := self.GetQuotaKeys()
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
 		return nil, err
 	}
 	pendingUsage.SetKeys(keys)
 	err = quotas.CheckSetPendingQuota(ctx, userCred, &pendingUsage)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
 		return nil, httperrors.NewOutOfQuotaError("Check set pending quota error %s", err)
 	}
 	regionKeys, err := self.GetRegionalQuotaKeys()
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
 		return nil, err
 	}
 	pendingRegionUsage.SetKeys(regionKeys)
 	err = quotas.CheckSetPendingQuota(ctx, userCred, &pendingRegionUsage)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
 		return nil, err
 	}
+	// migrate snapshotUsage into regionUsage, then discard snapshotUsage
 	pendingRegionUsage.Snapshot = snapshotUsage.Snapshot
 
 	instanceSnapshotName, err := db.GenerateName(InstanceSnapshotManager, self.GetOwnerId(),
 		fmt.Sprintf("%s-%s", newlyGuestName, rand.String(8)))
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
-		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, httperrors.NewInternalServerError("Generate snapshot name failed %s", err)
 	}
 	instanceSnapshot, err := InstanceSnapshotManager.CreateInstanceSnapshot(
 		ctx, userCred, self, instanceSnapshotName,
 		jsonutils.QueryBoolean(data, "auto_delete_instance_snapshot", false))
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
-		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err)
 	}
 
 	err = self.StartInstanceSnapshotAndCloneTask(
 		ctx, userCred, newlyGuestName, &pendingUsage, &pendingRegionUsage, instanceSnapshot, data.(*jsonutils.JSONDict))
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
-		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, err
 	}
 	return nil, nil

pkg/compute/models/guests.go

@@ -1360,7 +1360,7 @@ func (self *SGuest) PostUpdate(ctx context.Context, userCred mcclient.TokenCrede
 	if data.Contains("pending_usage") {
 		quota := SQuota{}
 		data.Unmarshal(&quota, "pending_usage")
-		quotas.CancelPendingUsage(ctx, userCred, &quota, &quota)
+		quotas.CancelPendingUsage(ctx, userCred, &quota, &quota, true)
 	}
 
 	self.StartSyncTask(ctx, userCred, true, "")
@@ -2580,7 +2580,7 @@ func (self *SGuest) attach2NetworkOnce(
 			log.Warningf("self.GetRegionalQuotaKeys fail %s", err)
 		}
 		cancelUsage.SetKeys(keys)
-		err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage)
+		err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
 		if err != nil {
 			log.Warningf("QuotaManager.CancelPendingUsage fail %s", err)
 		}
@@ -3192,7 +3192,7 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T
 		return nil, err
 	}
 	cancelUsage.SetKeys(keys)
-	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage)
+	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
 
 	return disk, nil
 }
@@ -3281,7 +3281,7 @@ func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcc
 		return err
 	}
 	cancelUsage.SetKeys(keys)
-	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage)
+	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true) // success
 	return err
 }
 

pkg/compute/models/snapshots.go

@@ -270,7 +270,7 @@ func (snapshot *SSnapshot) PostCreate(ctx context.Context, userCred mcclient.Tok
 	pendingUsage := SRegionQuota{Snapshot: 1}
 	keys := snapshot.GetQuotaKeys()
 	pendingUsage.SetKeys(keys)
-	err := quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+	err := quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
 	if err != nil {
 		log.Errorf("quotas.CancelPendingUsage fail %s", err)
 	}

pkg/compute/tasks/disk_base_task.go

@@ -40,7 +40,7 @@ func (self *SDiskBaseTask) finalReleasePendingUsage(ctx context.Context) {
 	pendingUsage := models.SQuota{}
 	err := self.GetPendingUsage(&pendingUsage, 0)
 	if err == nil && !pendingUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, false)
 	}
 }
 

pkg/compute/tasks/disk_batch_create_task.go

@@ -166,7 +166,7 @@ func (self *DiskBatchCreateTask) startCreateDisk(ctx context.Context, disk *mode
 		log.Warningf("disk.GetQuotaKeys fail %s", err)
 	}
 	quotaStorage.SetKeys(keys)
-	quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &quotaStorage)
+	quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &quotaStorage, true) // success
 	self.SetPendingUsage(&pendingUsage, 0)
 
 	disk.StartDiskCreateTask(ctx, self.GetUserCred(), false, "", self.GetTaskId())

pkg/compute/tasks/guest_base_task.go

@@ -40,11 +40,11 @@ func (self *SGuestBaseTask) finalReleasePendingUsage(ctx context.Context) {
 	pendingUsage := models.SQuota{}
 	err := self.GetPendingUsage(&pendingUsage, 0)
 	if err == nil && !pendingUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, false) // failure
 	}
 	pendingRegionUsage := models.SRegionQuota{}
 	err = self.GetPendingUsage(&pendingRegionUsage, 1)
 	if err == nil && !pendingRegionUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingRegionUsage, &pendingRegionUsage)
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingRegionUsage, &pendingRegionUsage, false) // failure
 	}
 }

pkg/compute/tasks/guest_batch_create_task.go

@@ -85,12 +85,12 @@ func (self *GuestBatchCreateTask) OnInit(ctx context.Context, objs []db.IStandal
 }
 
 func (self *GuestBatchCreateTask) OnScheduleFailCallback(ctx context.Context, obj IScheduleModel, reason string) {
-	self.SSchedTask.OnScheduleFailCallback(ctx, obj, reason)
 	guest := obj.(*models.SGuest)
 	if guest.DisableDelete.IsTrue() {
 		guest.SetDisableDelete(self.UserCred, false)
 	}
 	self.clearPendingUsage(ctx, guest)
+	self.SSchedTask.OnScheduleFailCallback(ctx, obj, reason)
 }
 
 func (self *GuestBatchCreateTask) SaveScheduleResultWithBackup(ctx context.Context, obj IScheduleModel, master, slave *schedapi.CandidateResource) {
@@ -140,7 +140,7 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest
 		log.Errorf("guest.GetQuotaKeys fail %s", err)
 	}
 	quotaCpuMem.SetKeys(keys)
-	err = quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &quotaCpuMem)
+	err = quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &quotaCpuMem, true) // success
 	self.SetPendingUsage(&pendingUsage, 0)
 
 	input, err := self.GetCreateInput()

pkg/compute/tasks/guest_change_config_task.go

@@ -242,7 +242,7 @@ func (self *GuestChangeConfigTask) OnGuestChangeCpuMemSpecComplete(ctx context.C
 	defer lockman.ReleaseClass(ctx, guest.GetModelManager(), guest.ProjectId)
 
 	if !cancelUsage.IsEmpty() {
-		err = quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &cancelUsage)
+		err = quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &cancelUsage, true) // success
 		if err != nil {
 			self.markStageFailed(ctx, guest, fmt.Sprintf("CancelPendingUsage fail %s", err))
 			return

pkg/compute/tasks/instance_snapshot_and_clone_task.go

@@ -68,13 +68,13 @@ func (self *InstanceSnapshotAndCloneTask) finalReleasePendingUsage(ctx context.C
 	pendingUsage := models.SQuota{}
 	err := self.GetPendingUsage(&pendingUsage, 0)
 	if err == nil && !pendingUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, false) // failure cleanup
 	}
 
 	pendingRegionUsage := models.SRegionQuota{}
 	err = self.GetPendingUsage(&pendingRegionUsage, 1)
 	if err == nil && !pendingRegionUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, false) // failure cleanup
 	}
 }
 

pkg/compute/tasks/instance_snapshot_create_task.go

@@ -49,7 +49,7 @@ func (self *InstanceSnapshotCreateTask) finalReleasePendingUsage(ctx context.Con
 	pendingUsage := models.SRegionQuota{}
 	err := self.GetPendingUsage(&pendingUsage, 0)
 	if err == nil && !pendingUsage.IsEmpty() {
-		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, self.UserCred, &pendingUsage, &pendingUsage, false) // final cleanup
 	}
 }
 

pkg/compute/tasks/pending_usage.go

@@ -17,6 +17,7 @@ package tasks
 import (
 	"context"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
@@ -36,11 +37,13 @@ func ClearTaskPendingUsage(ctx context.Context, task taskman.ITask) error {
 		return nil
 	}
 
+	log.Debugf("ClearTaskPendingUsage %s", jsonutils.Marshal(pendingUsage))
+
 	if pendingUsage.IsEmpty() {
 		return nil
 	}
 
-	err = quotas.CancelPendingUsage(ctx, task.GetUserCred(), &pendingUsage, &pendingUsage)
+	err = quotas.CancelPendingUsage(ctx, task.GetUserCred(), &pendingUsage, &pendingUsage, false)
 	if err != nil {
 		return errors.Wrap(err, "models.QuotaManager.CancelPendingUsage")
 	}
@@ -70,7 +73,7 @@ func ClearTaskPendingRegionUsage(ctx context.Context, task taskman.ITask) error
 		return nil
 	}
 
-	err = quotas.CancelPendingUsage(ctx, task.GetUserCred(), &pendingUsage, &pendingUsage)
+	err = quotas.CancelPendingUsage(ctx, task.GetUserCred(), &pendingUsage, &pendingUsage, false)
 	if err != nil {
 		return errors.Wrap(err, "models.QuotaManager.CancelPendingUsage")
 	}

pkg/image/models/image_guest.go

@@ -151,7 +151,7 @@ func (gi *SGuestImage) PostCreate(ctx context.Context, userCred mcclient.TokenCr
 	pendingUsage := SQuota{Image: int(imageNumber)}
 	keys := imageCreateInput2QuotaKeys(data, ownerId)
 	pendingUsage.SetKeys(keys)
-	quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+	quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
 
 	if !suc {
 		gi.SetStatus(userCred, api.IMAGE_STATUS_KILLED, "create subimage failed")

pkg/image/models/images.go

@@ -526,7 +526,7 @@ func (self *SImage) PostCreate(ctx context.Context, userCred mcclient.TokenCrede
 		cancelUsage := SQuota{Image: 1}
 		keys = self.GetQuotaKeys()
 		cancelUsage.SetKeys(keys)
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &cancelUsage)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &cancelUsage, true)
 	}
 
 	if data.Contains("properties") {