Instancebackup/syncstatus (#13618)

* feat(region): add status in snapshot or save phase for instancebackup

* feat(region): add sync_status for instancebackup

* fix(region): ignore uefi support check when related property is empty

* feat(region): add syncstatus for backup storage

* fix(region): correct stage for InstanceBackupDeleteTask

* feat(climc): add syncstatus for backupstorage and instancebackup

cmd/climc/shell/compute/backup.go

@@ -29,6 +29,7 @@ func init() {
 	bsCmd.Delete(&compute.BackupStorageIdOptions{})
 	bsCmd.Perform("public", &options.BasePublicOptions{})
 	bsCmd.Perform("private", &options.BaseIdOptions{})
+	bsCmd.Perform("syncstatus", &compute.DiskBackupSyncstatusOptions{})
 
 	dbCmd := shell.NewResourceCmd(&modules.DiskBackups)
 	dbCmd.List(&compute.DiskBackupListOptions{})
@@ -45,4 +46,5 @@ func init() {
 	ibCmd.Perform("recovery", &compute.InstanceBackupRecoveryOptions{})
 	ibCmd.Perform("pack", &compute.InstanceBackupPackOptions{})
 	ibCmd.PerformClass("create-from-package", &compute.InstanceBackupManagerCreateFromPackageOptions{})
+	ibCmd.Perform("syncstatus", &compute.DiskBackupSyncstatusOptions{})
 }

pkg/apis/compute/backup.go

@@ -21,8 +21,9 @@ import (
 )
 
 const (
-	BACKUPSTORAGE_TYPE_NFS      = "nfs"
-	BACKUPSTORAGE_STATUS_ONLINE = "online"
+	BACKUPSTORAGE_TYPE_NFS       = "nfs"
+	BACKUPSTORAGE_STATUS_ONLINE  = "online"
+	BACKUPSTORAGE_STATUS_OFFLINE = "offline"
 
 	BACKUP_STATUS_CREATING                = "creating"
 	BACKUP_STATUS_CREATE_FAILED           = "create_failed"
@@ -145,3 +146,6 @@ type InstanceBackupPackMetadata struct {
 	SizeMb         int
 	DiskMetadatas  []DiskBackupPackMetadata
 }
+
+type InstanceBackupManagerSyncstatusInput struct {
+}

pkg/apis/compute/instance_backup.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 import "yunion.io/x/onecloud/pkg/apis"
@@ -15,6 +29,11 @@ const (
 
 	INSTANCE_BACKUP_STATUS_CREATING_FROM_PACKAGE      = "creating_from_package"
 	INSTANCE_BACKUP_STATUS_CREATE_FROM_PACKAGE_FAILED = "create_from_package_failed"
+
+	INSTANCE_BACKUP_STATUS_SNAPSHOT        = "snapshot"
+	INSTANCE_BACKUP_STATUS_SNAPSHOT_FAILED = "snapshot_failed"
+	INSTANCE_BACKUP_STATUS_SAVING          = "saving"
+	INSTANCE_BACKUP_STATUS_SAVE_FAILED     = "save_failed"
 )
 
 type InstanceBackupListInput struct {

pkg/compute/models/backup.go

@@ -432,7 +432,7 @@ func (self *SDiskBackup) PerformSyncstatus(ctx context.Context, userCred mcclien
 		return nil, err
 	}
 	if count > 0 {
-		return nil, httperrors.NewBadRequestError("Snapshot has %d task active, can't sync status", count)
+		return nil, httperrors.NewBadRequestError("Backup has %d task active, can't sync status", count)
 	}
 
 	return nil, StartResourceSyncStatusTask(ctx, userCred, self, "DiskBackupSyncstatusTask", "")

pkg/compute/models/backup_storage.go

@@ -17,14 +17,17 @@ package models
 import (
 	"context"
 	"reflect"
+	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
@@ -123,6 +126,11 @@ func (bs *SBackupStorage) ValidateDeleteCondition(ctx context.Context, info json
 
 func (bs *SBackupStorage) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	bs.SEnabledStatusInfrasResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
+	err := StartResourceSyncStatusTask(ctx, userCred, bs, "BackupStorageSyncstatusTask", "")
+	if err != nil {
+		log.Errorf("unable to sync backup storage status")
+	}
+	bs.SetStatus(userCred, api.BACKUPSTORAGE_STATUS_OFFLINE, "")
 }
 
 func (bs *SBackupStorage) getMoreDetails(ctx context.Context, out api.BackupStorageDetails) api.BackupStorageDetails {
@@ -142,6 +150,10 @@ func (bm *SBackupStorageManager) FetchCustomizeColumns(ctx context.Context, user
 	return rows
 }
 
+func (self *SBackupStorage) GetRegionDriver() IRegionDriver {
+	return GetRegionDriver(api.CLOUD_PROVIDER_ONECLOUD)
+}
+
 func (bm *SBackupStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.BackupStorageListInput) (*sqlchemy.SQuery, error) {
 	var err error
 	q, err = bm.SEnabledStatusInfrasResourceBaseManager.ListItemFilter(ctx, q, userCred, input.EnabledStatusInfrasResourceBaseListInput)
@@ -150,3 +162,16 @@ func (bm *SBackupStorageManager) ListItemFilter(ctx context.Context, q *sqlchemy
 	}
 	return q, nil
 }
+
+func (self *SBackupStorage) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupSyncstatusInput) (jsonutils.JSONObject, error) {
+	var openTask = true
+	count, err := taskman.TaskManager.QueryTasksOfObject(self, time.Now().Add(-3*time.Minute), &openTask).CountWithError()
+	if err != nil {
+		return nil, err
+	}
+	if count > 0 {
+		return nil, httperrors.NewBadRequestError("Backup has %d task active, can't sync status", count)
+	}
+
+	return nil, StartResourceSyncStatusTask(ctx, userCred, self, "BackupStorageSyncstatusTask", "")
+}

pkg/compute/models/guests.go

@@ -1297,23 +1297,30 @@ func (manager *SGuestManager) validateCreateData(
 			input.OsArch = apis.OS_ARCH_AARCH64
 		}
 
-		imgSupportUEFI := imgProperties[imageapi.IMAGE_UEFI_SUPPORT] == "true"
+		var imgSupportUEFI *bool
+		if desc, ok := imgProperties[imageapi.IMAGE_UEFI_SUPPORT]; ok {
+			support := desc == "true"
+			imgSupportUEFI = &support
+		}
 		// imgIsWindows := imgProperties[imageapi.IMAGE_OS_TYPE] == "Windows"
 		// if imgSupportUEFI && imgIsWindows && len(input.IsolatedDevices) > 0 {
 		// 	input.Bios = "UEFI" // windows gpu passthrough
 		// }
 		if input.OsArch == apis.OS_ARCH_AARCH64 {
 			// arm image supports UEFI by default
-			imgSupportUEFI = true
+			support := true
+			imgSupportUEFI = &support
 		}
 
-		if imgSupportUEFI {
+		switch {
+		case imgSupportUEFI == nil:
+		case *imgSupportUEFI:
 			if len(input.Bios) == 0 {
 				input.Bios = "UEFI"
 			} else if input.Bios != "UEFI" {
 				return nil, httperrors.NewInputParameterError("UEFI image requires UEFI boot mode")
 			}
-		} else {
+		default:
 			// not UEFI or not detectable
 			if input.Bios == "UEFI" {
 				return nil, httperrors.NewInputParameterError("UEFI boot mode requires UEFI image")

pkg/compute/models/instance_backup.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -522,6 +523,19 @@ func (manager *SInstanceBackupManager) CreateInstanceBackupFromPackage(ctx conte
 	return ib, nil
 }
 
+func (ib *SInstanceBackup) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.InstanceBackupManagerSyncstatusInput) (jsonutils.JSONObject, error) {
+	var openTask = true
+	count, err := taskman.TaskManager.QueryTasksOfObject(ib, time.Now().Add(-3*time.Minute), &openTask).CountWithError()
+	if err != nil {
+		return nil, err
+	}
+	if count > 0 {
+		return nil, httperrors.NewBadRequestError("InstanceBackup has %d task active, can't sync status", count)
+	}
+
+	return nil, StartResourceSyncStatusTask(ctx, userCred, ib, "InstanceBackupSyncstatusTask", "")
+}
+
 func (ib *SInstanceBackup) FillFromPackMetadata(ctx context.Context, userCred mcclient.TokenCredential, diskBackupIds []string, metadata *api.InstanceBackupPackMetadata) (*SInstanceBackup, error) {
 	for i, backupId := range diskBackupIds {
 		_, err := DiskBackupManager.CreateFromPackMetadata(ctx, userCred, ib.BackupStorageId, backupId, fmt.Sprintf("%s_disk_%d", ib.Name, i), &metadata.DiskMetadatas[i])

pkg/compute/models/regiondrivers.go

@@ -122,6 +122,8 @@ type IRegionDriver interface {
 	RequestDeleteBackup(ctx context.Context, backup *SDiskBackup, task taskman.ITask) error
 	RequestCreateInstanceBackup(ctx context.Context, guest *SGuest, ib *SInstanceBackup, task taskman.ITask, params *jsonutils.JSONDict) error
 	RequestDeleteInstanceBackup(ctx context.Context, ib *SInstanceBackup, task taskman.ITask) error
+	RequestSyncInstanceBackupStatus(ctx context.Context, userCred mcclient.TokenCredential, ib *SInstanceBackup, task taskman.ITask) error
+	RequestSyncBackupStorageStatus(ctx context.Context, userCred mcclient.TokenCredential, bs *SBackupStorage, task taskman.ITask) error
 
 	RequestCreateInstanceSnapshot(ctx context.Context, guest *SGuest, isp *SInstanceSnapshot, task taskman.ITask, params *jsonutils.JSONDict) error
 	RequestDeleteInstanceSnapshot(ctx context.Context, isp *SInstanceSnapshot, task taskman.ITask) error

pkg/compute/regiondrivers/base.go

@@ -493,6 +493,14 @@ func (self *SBaseRegionDriver) ValidateCreateCdnData(ctx context.Context, userCr
 	return input, errors.Wrapf(cloudprovider.ErrNotImplemented, "ValidateCreateCdnData")
 }
 
+func (self *SBaseRegionDriver) RequestSyncInstanceBackupStatus(ctx context.Context, userCred mcclient.TokenCredential, ib *models.SInstanceBackup, task taskman.ITask) error {
+	return errors.Wrapf(cloudprovider.ErrNotImplemented, "SyncInstanceBackupStatus")
+}
+
+func (self *SBaseRegionDriver) RequestSyncBackupStorageStatus(ctx context.Context, userCred mcclient.TokenCredential, bs *models.SBackupStorage, task taskman.ITask) error {
+	return errors.Wrapf(cloudprovider.ErrNotImplemented, "SyncBackupStorageStatus")
+}
+
 func (self *SBaseRegionDriver) RequestPackInstanceBackup(ctx context.Context, ib *models.SInstanceBackup, task taskman.ITask, packageName string) error {
 	return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestPackInstanceBackup")
 }

pkg/compute/regiondrivers/kvm.go

@@ -1412,6 +1412,57 @@ func (self *SKVMRegionDriver) RequestUnpackInstanceBackup(ctx context.Context, i
 	return nil
 }
 
+func (self *SKVMRegionDriver) RequestSyncBackupStorageStatus(ctx context.Context, userCred mcclient.TokenCredential, bs *models.SBackupStorage, task taskman.ITask) error {
+	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+		host, err := models.HostManager.GetEnabledKvmHost()
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to GetEnabledKvmHost")
+		}
+		url := fmt.Sprintf("%s/storages/sync-backup-storage", host.ManagerUri)
+		body := jsonutils.NewDict()
+		body.Set("backup_storage_id", jsonutils.NewString(bs.GetId()))
+		body.Set("backup_storage_access_info", jsonutils.Marshal(bs.AccessInfo))
+		header := task.GetTaskRequestHeader()
+		_, res, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false)
+		if err != nil {
+			return nil, err
+		}
+		status, _ := res.GetString("status")
+		return nil, bs.SetStatus(userCred, status, "sync status")
+	})
+	return nil
+}
+
+func (self *SKVMRegionDriver) RequestSyncInstanceBackupStatus(ctx context.Context, userCred mcclient.TokenCredential, ib *models.SInstanceBackup, task taskman.ITask) error {
+	originStatus, _ := task.GetParams().GetString("origin_status")
+	if utils.IsInStringArray(originStatus, []string{api.INSTANCE_BACKUP_STATUS_CREATING, api.INSTANCE_BACKUP_STATUS_DELETING, api.INSTANCE_BACKUP_STATUS_RECOVERY, api.INSTANCE_BACKUP_STATUS_PACK, api.INSTANCE_BACKUP_STATUS_CREATING_FROM_PACKAGE, api.INSTANCE_BACKUP_STATUS_SAVING, api.INSTANCE_BACKUP_STATUS_SNAPSHOT}) {
+		err := ib.SetStatus(userCred, originStatus, "sync status")
+		if err != nil {
+			return err
+		}
+		task.SetStageComplete(ctx, nil)
+		return nil
+	}
+	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
+		task.SetStage("OnKvmBackupSyncstatus", nil)
+		backups, err := ib.GetBackups()
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to get backups")
+		}
+		for i := range backups {
+			params := jsonutils.NewDict()
+			params.Add(jsonutils.NewString(backups[i].GetStatus()), "origin_status")
+			task, err := taskman.TaskManager.NewTask(ctx, "DiskBackupSyncstatusTask", &backups[i], userCred, params, task.GetTaskId(), "", nil)
+			if err != nil {
+				return nil, err
+			}
+			task.ScheduleRun(nil)
+		}
+		return nil, nil
+	})
+	return nil
+}
+
 func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, userCred mcclient.TokenCredential, backup *models.SDiskBackup, task taskman.ITask) error {
 	taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
 		originStatus, _ := task.GetParams().GetString("origin_status")
@@ -1449,7 +1500,7 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u
 		if status == api.BACKUP_EXIST {
 			backupStatus = api.BACKUP_STATUS_READY
 		} else {
-			backupStatus = api.SNAPSHOT_UNKNOWN
+			backupStatus = api.BACKUP_STATUS_UNKNOWN
 		}
 		return nil, backup.SetStatus(userCred, backupStatus, "sync status")
 	})

pkg/compute/tasks/backup_storage_syncstatus_task.go

@@ -0,0 +1,44 @@
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type BackupStorageSyncstatusTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(BackupStorageSyncstatusTask{})
+}
+
+func (self *BackupStorageSyncstatusTask) taskFailed(ctx context.Context, bs *models.SBackupStorage, err jsonutils.JSONObject) {
+	logclient.AddActionLogWithContext(ctx, bs, logclient.ACT_SYNC_STATUS, err, self.UserCred, false)
+	self.SetStageFailed(ctx, err)
+}
+
+func (self *BackupStorageSyncstatusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	bs := obj.(*models.SBackupStorage)
+
+	self.SetStage("OnBackupStorageSyncStatus", nil)
+	err := bs.GetRegionDriver().RequestSyncBackupStorageStatus(ctx, self.GetUserCred(), bs, self)
+	if err != nil {
+		self.taskFailed(ctx, bs, jsonutils.NewString(err.Error()))
+		return
+	}
+}
+
+func (self *BackupStorageSyncstatusTask) OnBackupStorageSyncStatus(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}
+
+func (self *BackupStorageSyncstatusTask) OnBackupStorageSyncStatusFailed(ctx context.Context, backup *models.SBackupStorage, data jsonutils.JSONObject) {
+	self.taskFailed(ctx, backup, data)
+}

pkg/compute/tasks/instance_backup_create_task.go

@@ -36,12 +36,12 @@ func init() {
 	taskman.RegisterTask(InstanceBackupCreateTask{})
 }
 
-func (self *InstanceBackupCreateTask) taskFailed(ctx context.Context, ib *models.SInstanceBackup, guest *models.SGuest, reason jsonutils.JSONObject) {
+func (self *InstanceBackupCreateTask) taskFailed(ctx context.Context, ib *models.SInstanceBackup, guest *models.SGuest, reason jsonutils.JSONObject, status string) {
 	if guest != nil {
 		guest.SetStatus(self.UserCred, compute.VM_INSTANCE_BACKUP_FAILED, reason.String())
 	}
 	reasonStr, _ := reason.GetString()
-	ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_CREATE_FAILED, reasonStr)
+	ib.SetStatus(self.UserCred, status, reasonStr)
 	logclient.AddActionLogWithStartable(self, ib, logclient.ACT_CREATE, reason, self.UserCred, false)
 	self.SetStageFailed(ctx, reason)
 }
@@ -57,8 +57,9 @@ func (self *InstanceBackupCreateTask) OnInit(ctx context.Context, obj db.IStanda
 	self.SetStage("OnInstanceBackup", nil)
 	guest := models.GuestManager.FetchGuestById(ib.GuestId)
 	params := jsonutils.NewDict()
+	ib.SetStatus(self.GetUserCred(), compute.INSTANCE_BACKUP_STATUS_SNAPSHOT, "")
 	if err := ib.GetRegionDriver().RequestCreateInstanceBackup(ctx, guest, ib, self, params); err != nil {
-		self.taskFailed(ctx, ib, guest, jsonutils.NewString(err.Error()))
+		self.taskFailed(ctx, ib, guest, jsonutils.NewString(err.Error()), compute.INSTANCE_BACKUP_STATUS_SNAPSHOT_FAILED)
 	}
 }
 
@@ -70,33 +71,34 @@ func (self *InstanceBackupCreateTask) OnKvmDisksSnapshot(ctx context.Context, ib
 		log.Infof("subsTask %s result: %s", subTasks[i].SubtaskId, subTasks[i].Result)
 		result, err := jsonutils.ParseString(subTasks[i].Result)
 		if err != nil {
-			self.taskFailed(ctx, ib, guest, jsonutils.NewString(fmt.Sprintf("unable to parse %s", subTasks[i].Result)))
+			self.taskFailed(ctx, ib, guest, jsonutils.NewString(fmt.Sprintf("unable to parse %s", subTasks[i].Result)), compute.INSTANCE_BACKUP_STATUS_SNAPSHOT_FAILED)
 			return
 		}
 		if subTasks[i].Status == taskman.SUBTASK_FAIL {
-			self.taskFailed(ctx, ib, guest, result)
+			self.taskFailed(ctx, ib, guest, result, compute.INSTANCE_BACKUP_STATUS_SNAPSHOT_FAILED)
 			return
 		}
 		snapshotId, _ := result.GetString("snapshot_id")
 		diskBakcupId, _ := result.GetString("disk_backup_id")
 		ibackup, err := models.DiskBackupManager.FetchById(diskBakcupId)
 		if err != nil {
-			self.taskFailed(ctx, ib, guest, jsonutils.NewString(err.Error()))
+			self.taskFailed(ctx, ib, guest, jsonutils.NewString(err.Error()), compute.INSTANCE_BACKUP_STATUS_SNAPSHOT_FAILED)
 			return
 		}
 		backup := ibackup.(*models.SDiskBackup)
 		params := jsonutils.NewDict()
 		params.Set("snapshot_id", jsonutils.NewString(snapshotId))
 		if err := backup.StartBackupCreateTask(ctx, self.UserCred, params, self.Id); err != nil {
-			self.taskFailed(ctx, ib, guest, jsonutils.NewString(err.Error()))
+			self.taskFailed(ctx, ib, guest, jsonutils.NewString(err.Error()), compute.INSTANCE_BACKUP_STATUS_SAVE_FAILED)
 			return
 		}
 	}
+	ib.SetStatus(self.GetUserCred(), compute.INSTANCE_BACKUP_STATUS_SAVING, "")
 	guest.StartSyncstatus(ctx, self.UserCred, "")
 }
 
 func (self *InstanceBackupCreateTask) OnKvmDisksSnapshotFailed(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
-	self.taskFailed(ctx, ib, nil, data)
+	self.taskFailed(ctx, ib, nil, data, compute.INSTANCE_BACKUP_STATUS_SNAPSHOT_FAILED)
 }
 
 func (self *InstanceBackupCreateTask) OnInstanceBackup(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
@@ -107,15 +109,15 @@ func (self *InstanceBackupCreateTask) OnInstanceBackup(ctx context.Context, ib *
 		}
 		result, err := jsonutils.ParseString(subTasks[i].Result)
 		if err != nil {
-			self.taskFailed(ctx, ib, nil, jsonutils.NewString(fmt.Sprintf("unable to parse %s", subTasks[i].Result)))
+			self.taskFailed(ctx, ib, nil, jsonutils.NewString(fmt.Sprintf("unable to parse %s", subTasks[i].Result)), compute.INSTANCE_BACKUP_STATUS_SAVE_FAILED)
 			return
 		}
-		self.taskFailed(ctx, ib, nil, result)
+		self.taskFailed(ctx, ib, nil, result, compute.INSTANCE_BACKUP_STATUS_SAVE_FAILED)
 	}
 	// update size_mb
 	backups, err := ib.GetBackups()
 	if err != nil {
-		self.taskFailed(ctx, ib, nil, jsonutils.NewString(err.Error()))
+		self.taskFailed(ctx, ib, nil, jsonutils.NewString(err.Error()), compute.INSTANCE_BACKUP_STATUS_SAVE_FAILED)
 		return
 	}
 	var sizeMb int
@@ -130,5 +132,5 @@ func (self *InstanceBackupCreateTask) OnInstanceBackup(ctx context.Context, ib *
 }
 
 func (self *InstanceBackupCreateTask) OnInstanceBackupFailed(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
-	self.taskFailed(ctx, ib, nil, data)
+	self.taskFailed(ctx, ib, nil, data, compute.INSTANCE_BACKUP_STATUS_SAVE_FAILED)
 }

pkg/compute/tasks/instance_backup_delete_task.go

@@ -37,7 +37,7 @@ func init() {
 func (self *InstanceBackupDeleteTask) taskFailed(ctx context.Context, ib *models.SInstanceBackup, reason jsonutils.JSONObject) {
 	reasonStr, _ := reason.GetString()
 	ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_DELETE_FAILED, reasonStr)
-	logclient.AddActionLogWithStartable(self, ib, logclient.ACT_CREATE, reason, self.UserCred, false)
+	logclient.AddActionLogWithStartable(self, ib, logclient.ACT_DELETE, reason, self.UserCred, false)
 	self.SetStageFailed(ctx, reason)
 }
 
@@ -79,14 +79,13 @@ func (self *InstanceBackupDeleteTask) OnKvmDiskBackupDelete(
 	}
 }
 
-func (self *InstanceBackupDeleteTask) OnKvmBackupDeleteFailed(
+func (self *InstanceBackupDeleteTask) OnKvmDiskBackupDeleteFailed(
 	ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
 	self.taskFailed(ctx, ib, data)
 }
 
 func (self *InstanceBackupDeleteTask) OnInstanceBackupDelete(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
 	self.taskSuccess(ctx, ib, data)
-
 }
 
 func (self *InstanceBackupDeleteTask) OnInstanceBackupDeleteFailed(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {

pkg/compute/tasks/instance_backup_syncstatus.go

@@ -0,0 +1,109 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tasks
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type InstanceBackupSyncstatusTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(InstanceBackupSyncstatusTask{})
+}
+
+func (self *InstanceBackupSyncstatusTask) taskFailed(ctx context.Context, ib *models.SInstanceBackup, err jsonutils.JSONObject) {
+	logclient.AddActionLogWithContext(ctx, ib, logclient.ACT_SYNC_STATUS, err, self.UserCred, false)
+	self.SetStageFailed(ctx, err)
+}
+
+func (self *InstanceBackupSyncstatusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	ib := obj.(*models.SInstanceBackup)
+	self.SetStage("OnInstnaceBackupSyncstatus", nil)
+	rd := ib.GetRegionDriver()
+	err := rd.RequestSyncInstanceBackupStatus(ctx, self.GetUserCred(), ib, self)
+	if err != nil {
+		self.taskFailed(ctx, ib, jsonutils.NewString(err.Error()))
+	}
+	return
+}
+
+func (self *InstanceBackupSyncstatusTask) OnKvmBackupSyncstatus(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
+	subTasks := taskman.SubTaskManager.GetTotalSubtasks(self.Id, "OnKvmDisksSnapshot", "")
+	for i := range subTasks {
+		log.Infof("subsTask %s result: %s", subTasks[i].SubtaskId, subTasks[i].Result)
+		result, err := jsonutils.ParseString(subTasks[i].Result)
+		if err != nil {
+			self.taskFailed(ctx, ib, jsonutils.NewString(fmt.Sprintf("unable to parse %s", subTasks[i].Result)))
+			return
+		}
+		if subTasks[i].Status == taskman.SUBTASK_FAIL {
+			self.taskFailed(ctx, ib, result)
+			return
+		}
+	}
+	backups, err := ib.GetBackups()
+	if err != nil {
+		self.taskFailed(ctx, ib, jsonutils.NewString(err.Error()))
+		return
+	}
+	var status string
+	for i := range backups {
+		if backups[i].Status == compute.BACKUP_STATUS_UNKNOWN {
+			status = compute.BACKUP_STATUS_UNKNOWN
+			break
+		}
+		if backups[i].Status != compute.BACKUP_STATUS_READY {
+			status = ""
+			break
+		}
+		status = compute.BACKUP_STATUS_READY
+	}
+	if status == "" {
+		originStatus, _ := self.Params.GetString("origin_status")
+		ib.SetStatus(self.GetUserCred(), originStatus, "")
+	} else {
+		ib.SetStatus(self.GetUserCred(), status, "")
+	}
+	self.OnInstnaceBackupSyncstatus(ctx, ib, data)
+	return
+}
+
+func (self *InstanceBackupSyncstatusTask) OnKvmBackupSyncstatusFailed(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
+	self.taskFailed(ctx, ib, data)
+	return
+}
+
+func (self *InstanceBackupSyncstatusTask) OnInstnaceBackupSyncstatus(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+	return
+}
+
+func (self *InstanceBackupSyncstatusTask) OnInstnaceBackupSyncstatusFailed(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
+	self.taskFailed(ctx, ib, data)
+	return
+}

pkg/hostman/storageman/backupstorage/backup_storage.go

@@ -33,6 +33,7 @@ type IBackupStorage interface {
 	ConvertFrom(srcPath string, format qemuimg.TImageFormat, backupId string) (int, error)
 	InstancePack(packageName string, backupIds []string, metadata *api.InstanceBackupPackMetadata) error
 	InstanceUnpack(packageName string) ([]string, *api.InstanceBackupPackMetadata, error)
+	IsOnline() (bool, error)
 }
 
 var backupStoragePool *sync.Map = &sync.Map{}

pkg/hostman/storageman/backupstorage/backup_storage_nfs.go

@@ -37,6 +37,8 @@ import (
 
 const BackupStoragePath = "/opt/cloud/workspace/backupstorage"
 
+var ErrorBackupStorageOffline error = errors.Error("backup storage offline")
+
 type SNFSBackupStorage struct {
 	BackupStorageId string
 	Path            string
@@ -83,7 +85,7 @@ func (s *SNFSBackupStorage) checkAndMount() error {
 	err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx,
 		"mount", "-t", "nfs", fmt.Sprintf("%s:%s", s.NfsHost, s.NfsSharedDir), s.Path).Run()
 	if err != nil {
-		return err
+		return ErrorBackupStorageOffline
 	}
 	backupDir := s.getBackupDir()
 	if !fileutils2.Exists(backupDir) {
@@ -463,3 +465,15 @@ func (s *SNFSBackupStorage) IsExists(backupId string) (bool, error) {
 	filename := path.Join(backupDir, backupId)
 	return fileutils2.Exists(filename), nil
 }
+
+func (s *SNFSBackupStorage) IsOnline() (bool, error) {
+	err := s.checkAndMount()
+	if errors.Cause(err) == ErrorBackupStorageOffline {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	s.unMount()
+	return true, nil
+}

pkg/hostman/storageman/storagehandler/storagehandler.go

@@ -71,6 +71,9 @@ func AddStorageHandler(prefix string, app *appsrv.Application) {
 		app.AddHandler("POST",
 			fmt.Sprintf("%s/%s/unpack-instance-backup", prefix, keyWords),
 			auth.Authenticate(storageUnpackInstanceBackup))
+		app.AddHandler("POST",
+			fmt.Sprintf("%s/%s/sync-backup-storage", prefix, keyWords),
+			auth.Authenticate(storageSyncBackupStorage))
 	}
 }
 
@@ -240,6 +243,41 @@ func storageSyncBackup(ctx context.Context, w http.ResponseWriter, r *http.Reque
 	hostutils.Response(ctx, w, ret)
 }
 
+func storageSyncBackupStorage(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	_, _, body := appsrv.FetchEnv(ctx, w, r)
+	backupStorageId, err := body.GetString("backup_storage_id")
+	if err != nil {
+		hostutils.Response(ctx, w, httperrors.NewMissingParameterError("backup_storage_id"))
+		return
+	}
+	backupStorageAccessInfo, err := body.Get("backup_storage_access_info")
+	if err != nil {
+		hostutils.Response(ctx, w, httperrors.NewMissingParameterError("backup_storage_access_info"))
+		return
+	}
+	backupStorage, err := backupstorage.GetBackupStorage(backupStorageId, backupStorageAccessInfo.(*jsonutils.JSONDict))
+	if err != nil {
+		hostutils.Response(ctx, w, err)
+		return
+	}
+	exist, err := backupStorage.IsOnline()
+	if err != nil {
+		hostutils.Response(ctx, w, err)
+		return
+	}
+	var (
+		ret    = jsonutils.NewDict()
+		status string
+	)
+	if exist {
+		status = compute.BACKUPSTORAGE_STATUS_ONLINE
+	} else {
+		status = compute.BACKUPSTORAGE_STATUS_OFFLINE
+	}
+	ret.Set("status", jsonutils.NewString(status))
+	hostutils.Response(ctx, w, ret)
+}
+
 func storagePackInstanceBackup(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	_, _, body := appsrv.FetchEnv(ctx, w, r)
 	if !checkOptions(ctx, w, body, "package_name", "backup_ids", "backup_storage_id", "backup_storage_access_info", "metadata") {