Merge pull request #2469 from swordqiu/hotfix/qj-resolve-guest-resize-disk-deadlock

fix: disk-resize sync guest status cause deadlock
This commit is contained in:
yunion-ci-robot
2019-08-22 09:44:06 +08:00
committed by GitHub
11 changed files with 206 additions and 120 deletions
+15
View File
@@ -1001,4 +1001,19 @@ func init() {
printObject(result)
return nil
})
type ServerResizeDiskOptions struct {
Server string `help:"ID or name of VM" json:"-" optional:"false" positional:"true"`
Disk string `help:"ID or name of disk to resize" json:"disk" optional:"false" positional:"true"`
Size string `help:"new size of disk in MB" json:"size" optional:"false" positional:"true"`
}
R(&ServerResizeDiskOptions{}, "server-resize-disk", "Resize attached disk of a server", func(s *mcclient.ClientSession, args *ServerResizeDiskOptions) error {
params := jsonutils.Marshal(args)
result, err := modules.Servers.PerformAction(s, args.Server, "resize-disk", params)
if err != nil {
return err
}
printObject(result)
return nil
})
}
+5 -3
View File
@@ -88,11 +88,13 @@ const (
VM_SYNC_CONFIG = "sync_config"
VM_SYNC_FAIL = "sync_fail"
VM_START_RESIZE_DISK = "start_resize_disk"
VM_RESIZE_DISK = "resize_disk"
VM_RESIZE_DISK_FAILED = "resize_disk_fail"
VM_START_SAVE_DISK = "start_save_disk"
VM_SAVE_DISK = "save_disk"
VM_SAVE_DISK_FAILED = "save_disk_failed"
VM_START_SAVE_DISK = "start_save_disk"
VM_SAVE_DISK = "save_disk"
VM_SAVE_DISK_FAILED = "save_disk_failed"
VM_RESTORING_SNAPSHOT = "restoring_snapshot"
VM_RESTORE_DISK = "restore_disk"
+1 -1
View File
@@ -62,7 +62,7 @@ func (self *SBaremetalHostDriver) RequestRebuildDiskOnStorage(ctx context.Contex
return fmt.Errorf("not supported")
}
func (self *SBaremetalHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, guest *models.SGuest, sizeMb int64, task taskman.ITask) error {
func (self *SBaremetalHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, sizeMb int64, task taskman.ITask) error {
return fmt.Errorf("not supported")
}
+2 -28
View File
@@ -294,13 +294,14 @@ func (driver *SKVMHostDriver) RequestDeallocateBackupDiskOnHost(ctx context.Cont
return err
}
func (self *SKVMHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, guest *models.SGuest, sizeMb int64, task taskman.ITask) error {
func (self *SKVMHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, sizeMb int64, task taskman.ITask) error {
header := task.GetTaskRequestHeader()
url := fmt.Sprintf("/disks/%s/resize/%s", storage.Id, disk.Id)
body := jsonutils.NewDict()
content := jsonutils.NewDict()
content.Add(jsonutils.NewInt(sizeMb), "size")
guest := disk.GetGuest()
if guest != nil {
content.Add(jsonutils.NewString(guest.Id), "server_id")
}
@@ -309,33 +310,6 @@ func (self *SKVMHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *m
return err
}
/*
func (self *SKVMHostDriver) RequestResizeDiskOnHostOnline(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, sizeMb int64, task taskman.ITask) error {
header := task.GetTaskRequestHeader()
guests := disk.GetAttachedGuests()
if len(guests) == 0 {
return fmt.Errorf("no valid guest")
}
if len(guests) > 1 {
return fmt.Errorf("cannot resize across more than 1 guest")
}
guest := guests[0]
guestdisk := guest.GetGuestDisk(disk.GetId())
url := fmt.Sprintf("/servers/%s/monitor", guest.GetId())
body := jsonutils.NewDict()
cmd := fmt.Sprintf("block_resize drive_%d %dM", guestdisk.Index, sizeMb)
body.Add(jsonutils.NewString(cmd), "cmd")
taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
return host.Request(ctx, task.GetUserCred(), "POST", url, header, body)
})
return nil
}
*/
func (self *SKVMHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error {
body := jsonutils.NewDict()
body.Add(jsonutils.Marshal(map[string]string{"image_id": imageId}), "disk")
+1 -1
View File
@@ -213,7 +213,7 @@ func (self *SManagedVirtualizationHostDriver) RequestSaveUploadImageOnHost(ctx c
return nil
}
func (self *SManagedVirtualizationHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, guest *models.SGuest, sizeMb int64, task taskman.ITask) error {
func (self *SManagedVirtualizationHostDriver) RequestResizeDiskOnHost(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, sizeMb int64, task taskman.ITask) error {
iCloudStorage, err := storage.GetIStorage()
if err != nil {
log.Errorf("storage.GetIStorage fail %s", err)
+47 -34
View File
@@ -298,6 +298,7 @@ func (self *SDisk) GetGuestdisks() []SGuestdisk {
}
return guestdisks
}
func (self *SDisk) GetGuests() []SGuest {
result := make([]SGuest, 0)
query := GuestManager.Query()
@@ -314,6 +315,14 @@ func (self *SDisk) GetGuests() []SGuest {
return result
}
func (self *SDisk) GetGuest() *SGuest {
guests := self.GetGuests()
if len(guests) > 0 {
return &guests[0]
}
return nil
}
func (self *SDisk) GetGuestsCount() (int, error) {
guests := GuestManager.Query().SubQuery()
guestdisks := GuestdiskManager.Query().SubQuery()
@@ -708,54 +717,60 @@ func (self *SDisk) AllowPerformResize(ctx context.Context, userCred mcclient.Tok
return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "resize")
}
func (self *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
sizeStr, err := data.GetString("size")
if err != nil {
return nil, httperrors.NewMissingParameterError("size")
}
sizeMb, err := fileutils.GetSizeMb(sizeStr, 'M', 1024)
func (disk *SDisk) PerformResize(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
guest := disk.GetGuest()
err := disk.doResize(ctx, userCred, data, guest)
if err != nil {
return nil, err
}
if self.Status != api.DISK_READY {
return nil, httperrors.NewResourceNotReadyError("Resize disk when disk is READY")
return nil, nil
}
func (disk *SDisk) doResize(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, guest *SGuest) error {
sizeStr, err := data.GetString("size")
if err != nil {
return httperrors.NewMissingParameterError("size")
}
if sizeMb < self.DiskSize {
return nil, httperrors.NewUnsupportOperationError("Disk cannot be thrink")
sizeMb, err := fileutils.GetSizeMb(sizeStr, 'M', 1024)
if err != nil {
return err
}
if sizeMb == self.DiskSize {
return nil, nil
if disk.Status != api.DISK_READY {
return httperrors.NewResourceNotReadyError("Resize disk when disk is READY")
}
addDisk := sizeMb - self.DiskSize
storage := self.GetStorage()
if sizeMb < disk.DiskSize {
return httperrors.NewUnsupportOperationError("Disk cannot be thrink")
}
if sizeMb == disk.DiskSize {
return nil
}
addDisk := sizeMb - disk.DiskSize
storage := disk.GetStorage()
if host := storage.GetMasterHost(); host != nil {
if err := host.GetHostDriver().ValidateDiskSize(storage, sizeMb>>10); err != nil {
return nil, httperrors.NewInputParameterError(err.Error())
return httperrors.NewInputParameterError(err.Error())
}
}
if int64(addDisk) > storage.GetFreeCapacity() && !storage.IsEmulated {
return nil, httperrors.NewOutOfResourceError("Not enough free space")
return httperrors.NewOutOfResourceError("Not enough free space")
}
if guests := self.GetGuests(); len(guests) > 0 {
if err := guests[0].ValidateResizeDisk(self, storage); err != nil {
return nil, httperrors.NewInputParameterError(err.Error())
if guest != nil {
if err := guest.ValidateResizeDisk(disk, storage); err != nil {
return httperrors.NewInputParameterError(err.Error())
}
}
pendingUsage := SQuota{Storage: int(addDisk)}
quotaName := storage.getQuotaPlatformID()
if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, userCred, quotaName, &pendingUsage); err != nil {
return nil, httperrors.NewOutOfQuotaError(err.Error())
return httperrors.NewOutOfQuotaError(err.Error())
}
guests := self.GetGuests()
var guest *SGuest
if len(guests) == 1 {
guest = &guests[0]
if guest != nil {
return guest.StartGuestDiskResizeTask(ctx, userCred, disk.Id, int64(sizeMb), "", &pendingUsage)
} else {
return disk.StartDiskResizeTask(ctx, userCred, int64(sizeMb), "", &pendingUsage)
}
return nil, self.StartDiskResizeTask(ctx, userCred, int64(sizeMb), "", &pendingUsage, guest)
}
func (self *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) {
@@ -1612,17 +1627,15 @@ func (self *SDisk) GetCustomizeColumns(ctx context.Context, userCred mcclient.To
return self.getMoreDetails(extra)
}
func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, sizeMb int64, parentTaskId string, pendingUsage quotas.IQuota, guest *SGuest) error {
func (self *SDisk) StartDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, sizeMb int64, parentTaskId string, pendingUsage quotas.IQuota) error {
self.SetStatus(userCred, api.DISK_START_RESIZE, "StartDiskResizeTask")
params := jsonutils.NewDict()
params.Add(jsonutils.NewInt(sizeMb), "size")
if guest != nil {
params.Add(jsonutils.NewString(guest.Id), "guest_id")
}
if task, err := taskman.TaskManager.NewTask(ctx, "DiskResizeTask", self, userCred, params, parentTaskId, "", pendingUsage); err != nil {
task, err := taskman.TaskManager.NewTask(ctx, "DiskResizeTask", self, userCred, params, parentTaskId, "", pendingUsage)
if err != nil {
return err
} else {
task.ScheduleRun(nil)
}
task.ScheduleRun(nil)
return nil
}
+43 -1
View File
@@ -1968,7 +1968,6 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T
disks := self.GetDisks()
var addDisk int
var diskIdx = 1
var newDiskIdx = 0
var diskSizes = make(map[string]int, 0)
var newDisks = make([]*api.DiskConfig, 0)
@@ -1981,6 +1980,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T
}
}
var diskIdx = 1
for _, diskConf := range inputDisks {
diskConf, err = parseDiskInfo(ctx, userCred, diskConf)
if err != nil {
@@ -3432,3 +3432,45 @@ func (guest *SGuest) PerformChangeOwner(ctx context.Context, userCred mcclient.T
}
return guest.SVirtualResourceBase.PerformChangeOwner(ctx, userCred, query, data)
}
func (guest *SGuest) AllowPerformResizeDisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
return guest.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, guest, "resize-disk")
}
func (guest *SGuest) PerformResizeDisk(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
diskStr, _ := data.GetString("disk")
if len(diskStr) == 0 {
return nil, httperrors.NewMissingParameterError("disk")
}
diskObj, err := DiskManager.FetchByIdOrName(userCred, diskStr)
if err != nil {
if err == sql.ErrNoRows {
return nil, httperrors.NewResourceNotFoundError2(DiskManager.Keyword(), diskStr)
} else {
return nil, httperrors.NewGeneralError(err)
}
}
guestdisk := guest.GetGuestDisk(diskObj.GetId())
if guestdisk == nil {
return nil, httperrors.NewInvalidStatusError("disk %s not attached to server", diskStr)
}
disk := diskObj.(*SDisk)
err = disk.doResize(ctx, userCred, data, guest)
if err != nil {
return nil, err
}
return nil, nil
}
func (guest *SGuest) StartGuestDiskResizeTask(ctx context.Context, userCred mcclient.TokenCredential, diskId string, sizeMb int64, parentTaskId string, pendingUsage quotas.IQuota) error {
guest.SetStatus(userCred, api.VM_START_RESIZE_DISK, "StartGuestDiskResizeTask")
params := jsonutils.NewDict()
params.Add(jsonutils.NewInt(sizeMb), "size")
params.Add(jsonutils.NewString(diskId), "disk_id")
task, err := taskman.TaskManager.NewTask(ctx, "GuestResizeDiskTask", guest, userCred, params, parentTaskId, "", pendingUsage)
if err != nil {
return err
}
task.ScheduleRun(nil)
return nil
}
+1 -3
View File
@@ -43,9 +43,7 @@ type IHostDriver interface {
RequestDeallocateDiskOnHost(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error
RequestDeallocateBackupDiskOnHost(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, task taskman.ITask) error
//RequestResizeDiskOnHostOnline(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error
RequestResizeDiskOnHost(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, guest *SGuest, size int64, task taskman.ITask) error
RequestResizeDiskOnHost(ctx context.Context, host *SHost, storage *SStorage, disk *SDisk, size int64, task taskman.ITask) error
RequestDeleteSnapshotsWithStorage(ctx context.Context, host *SHost, snapshot *SSnapshot, task taskman.ITask) error
RequestResetDisk(ctx context.Context, host *SHost, disk *SDisk, params *jsonutils.JSONDict, task taskman.ITask) error
+6 -47
View File
@@ -39,31 +39,17 @@ func init() {
func (self *DiskResizeTask) SetDiskReady(ctx context.Context, disk *models.SDisk, userCred mcclient.TokenCredential, reason string) {
// 此函数主要避免虚机更改配置时,虚机可能出现中间状态
if self.HasParentTask() {
// 若是子任务,磁盘关联的虚拟机状态由父任务恢复,仅恢复磁盘自身状态即可
disk.SetStatus(userCred, api.DISK_READY, reason)
} else {
// 若不是子任务,由于扩容时设置了关联的虚机状态,虚机的状态也由自己恢复
disk.SetDiskReady(ctx, userCred, reason)
}
// 若是子任务,磁盘关联的虚拟机状态由父任务恢复,仅恢复磁盘自身状态即可
disk.SetStatus(userCred, api.DISK_READY, reason)
// 若不是子任务,由于扩容时设置了关联的虚机状态,虚机的状态也由自己恢复
}
func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
disk := obj.(*models.SDisk)
guestId, _ := self.Params.GetString("guest_id")
var masterGuest *models.SGuest
if len(guestId) > 0 {
masterGuest = models.GuestManager.FetchGuestById(guestId)
}
storage := disk.GetStorage()
host := storage.GetMasterHost()
if masterGuest != nil {
host = masterGuest.GetHost()
}
reason := "Cannot find host for disk"
if host == nil || host.HostStatus != api.HOST_ONLINE {
self.SetDiskReady(ctx, disk, self.GetUserCred(), reason)
@@ -74,17 +60,14 @@ func (self *DiskResizeTask) OnInit(ctx context.Context, obj db.IStandaloneModel,
}
disk.SetStatus(self.GetUserCred(), api.DISK_START_RESIZE, "")
if masterGuest != nil {
masterGuest.SetStatus(self.GetUserCred(), api.VM_RESIZE_DISK, "")
}
self.StartResizeDisk(ctx, host, storage, disk, masterGuest)
self.StartResizeDisk(ctx, host, storage, disk)
}
func (self *DiskResizeTask) StartResizeDisk(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, guest *models.SGuest) {
func (self *DiskResizeTask) StartResizeDisk(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk) {
log.Infof("Resizing disk on host %s ...", host.GetName())
self.SetStage("OnDiskResizeComplete", nil)
sizeMb, _ := self.GetParams().Int("size")
if err := host.GetHostDriver().RequestResizeDiskOnHost(ctx, host, storage, disk, guest, sizeMb, self); err != nil {
if err := host.GetHostDriver().RequestResizeDiskOnHost(ctx, host, storage, disk, sizeMb, self); err != nil {
log.Errorf("request_resize_disk_on_host: %v", err)
self.OnStartResizeDiskFailed(ctx, disk, err)
return
@@ -137,37 +120,13 @@ func (self *DiskResizeTask) OnDiskResizeComplete(ctx context.Context, disk *mode
}
func (self *DiskResizeTask) OnDiskResized(ctx context.Context, disk *models.SDisk) {
guestId, _ := self.Params.GetString("guest_id")
if len(guestId) > 0 {
self.SetStage("TaskComplete", nil)
masterGuest := models.GuestManager.FetchGuestById(guestId)
if self.HasParentTask() {
masterGuest.StartSyncTaskWithoutSyncstatus(ctx, self.UserCred, false, self.GetId())
} else {
masterGuest.StartSyncTask(ctx, self.UserCred, false, self.GetId())
}
} else {
self.TaskComplete(ctx, disk, nil)
}
}
func (self *DiskResizeTask) TaskComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) {
self.SetStageComplete(ctx, disk.GetShortDesc(ctx))
self.finalReleasePendingUsage(ctx)
}
func (self *DiskResizeTask) TaskCompleteFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) {
self.SetStageFailed(ctx, data.String())
}
func (self *DiskResizeTask) OnDiskResizeCompleteFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) {
self.SetDiskReady(ctx, disk, self.GetUserCred(), data.String())
db.OpsLog.LogEvent(disk, db.ACT_RESIZE_FAIL, disk.GetShortDesc(ctx), self.UserCred)
logclient.AddActionLogWithStartable(self, disk, logclient.ACT_RESIZE, data.String(), self.UserCred, false)
guestId, _ := self.Params.GetString("guest_id")
if len(guestId) > 0 {
masterGuest := models.GuestManager.FetchGuestById(guestId)
masterGuest.SetStatus(self.UserCred, api.VM_RESIZE_DISK_FAILED, data.String())
}
self.SetStageFailed(ctx, data.String())
}
@@ -98,9 +98,9 @@ func (self *GuestChangeConfigTask) OnDisksResizeComplete(ctx context.Context, ob
self.markStageFailed(ctx, guest, fmt.Sprintf("self.GetPendingUsage(&pendingUsage) fail %s", err))
return
}
err = disk.StartDiskResizeTask(ctx, self.UserCred, size, self.GetTaskId(), &pendingUsage, guest)
err = guest.StartGuestDiskResizeTask(ctx, self.UserCred, disk.Id, size, self.GetTaskId(), &pendingUsage)
if err != nil {
self.markStageFailed(ctx, guest, fmt.Sprintf("disk.StartDiskResizeTask fail %s", err))
self.markStageFailed(ctx, guest, fmt.Sprintf("guest.StartGuestDiskResizeTask fail %s", err))
return
}
return
@@ -0,0 +1,83 @@
package tasks
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/util/logclient"
)
type GuestResizeDiskTask struct {
SGuestBaseTask
}
func init() {
taskman.RegisterTask(GuestResizeDiskTask{})
}
func (task *GuestResizeDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
guest := obj.(*models.SGuest)
guest.SetStatus(task.GetUserCred(), api.VM_RESIZE_DISK, "")
db.OpsLog.LogEvent(guest, db.ACT_RESIZING, task.Params, task.UserCred)
diskId, _ := task.Params.GetString("disk_id")
sizeMb, _ := task.Params.Int("size")
diskObj, err := models.DiskManager.FetchById(diskId)
if err != nil {
task.OnTaskFailed(ctx, guest, err.Error())
return
}
pendingUsage := models.SQuota{}
err = task.GetPendingUsage(&pendingUsage)
if err != nil {
task.OnTaskFailed(ctx, guest, err.Error())
return
}
task.SetStage("OnDiskResizeComplete", nil)
diskObj.(*models.SDisk).StartDiskResizeTask(ctx, task.UserCred, sizeMb, task.GetId(), &pendingUsage)
}
func (task *GuestResizeDiskTask) OnTaskFailed(ctx context.Context, guest *models.SGuest, reason string) {
log.Errorf("GuestResizeDiskTask fail: %s", reason)
guest.SetStatus(task.UserCred, api.VM_RESIZE_DISK_FAILED, reason)
db.OpsLog.LogEvent(guest, db.ACT_RESIZE_FAIL, reason, task.UserCred)
logclient.AddActionLogWithStartable(task, guest, logclient.ACT_RESIZE, reason, task.UserCred, false)
task.SetStageFailed(ctx, reason)
}
func (task *GuestResizeDiskTask) OnDiskResizeComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
guest := obj.(*models.SGuest)
db.OpsLog.LogEvent(guest, db.ACT_RESIZE, task.Params, task.UserCred)
logclient.AddActionLogWithStartable(task, guest, logclient.ACT_RESIZE, task.Params, task.UserCred, true)
task.SetStage("TaskComplete", nil)
if task.HasParentTask() {
guest.StartSyncTaskWithoutSyncstatus(ctx, task.UserCred, false, task.GetId())
} else {
guest.StartSyncTask(ctx, task.UserCred, false, task.GetId())
}
}
func (task *GuestResizeDiskTask) OnDiskResizeCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
guest := obj.(*models.SGuest)
task.OnTaskFailed(ctx, guest, data.String())
}
func (task *GuestResizeDiskTask) TaskComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
guest := obj.(*models.SGuest)
task.SetStageComplete(ctx, guest.GetShortDesc(ctx))
}
func (task *GuestResizeDiskTask) TaskCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
task.SetStageFailed(ctx, data.String())
}