fix: export/import disk backup (#23759)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

cmd/climc/shell/compute/backup.go

@@ -15,6 +15,11 @@
 package compute
 
 import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/jsonutils"
+
 	"yunion.io/x/onecloud/cmd/climc/shell"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -39,6 +44,16 @@ func init() {
 	dbCmd.Create(&compute.DiskBackupCreateOptions{})
 	dbCmd.Perform("recovery", &compute.DiskBackupRecoveryOptions{})
 	dbCmd.Perform("syncstatus", &compute.DiskBackupSyncstatusOptions{})
+	dbCmd.GetWithCustomOptionShow("export-info", func(info jsonutils.JSONObject, opts shell.IGetOpt) {
+		args := opts.(*compute.DiskBackupExportOptions)
+		if args.Output != "" {
+			os.WriteFile(args.Output, []byte(info.String()), 0644)
+			fmt.Printf("Exported to %s\n", args.Output)
+		} else {
+			printObject(info)
+		}
+	}, &compute.DiskBackupExportOptions{})
+	dbCmd.PerformClass("import", &compute.DiskBackupImportOptions{})
 
 	ibCmd := shell.NewResourceCmd(&modules.InstanceBackups)
 	ibCmd.List(&compute.InstanceBackupListOptions{})

pkg/apis/compute/backup.go

@@ -43,6 +43,9 @@ const (
 	BACKUP_STATUS_RECOVERY                = "recovery"
 	BACKUP_STATUS_RECOVERY_FAILED         = "recovery_failed"
 	BACKUP_STATUS_UNKNOWN                 = "unknown"
+	BACKUP_STATUS_START_IMPORT            = "start_import"
+	BACKUP_STATUS_IMPORTING               = "importing"
+	BACKUP_STATUS_IMPORT_FAILED           = "import_failed"
 
 	BACKUP_EXIST     = "exist"
 	BACKUP_NOT_EXIST = "not_exist"
@@ -163,6 +166,16 @@ type DiskBackupPackMetadata struct {
 	DiskConfig *SBackupDiskConfig
 }
 
+type DiskBackupExportInfo struct {
+	DiskBackupPackMetadata
+
+	DiskBackupImportTaskInput
+}
+
+type DiskBackupImportTaskInput struct {
+	AccessUrl string
+}
+
 type InstanceBackupPackMetadata struct {
 	OsArch         string
 	ServerConfig   jsonutils.JSONObject
@@ -201,6 +214,8 @@ type SBackupStorageAccessInfo struct {
 	ObjectSecret string `json:"object_secret"`
 	// description: signing version, can be v2/v4, default is v4
 	ObjectSignVer string `json:"object_sign_ver"`
+	// description: external access url of object storage bucket
+	ObjectBucketUrlExt string `json:"object_bucket_url_ext"`
 }
 
 func (ba *SBackupStorageAccessInfo) String() string {
@@ -219,3 +234,15 @@ type ServerCreateInstanceBackupInput struct {
 	// 备份存储ID
 	BackupStorageId string `json:"backup_storage_id"`
 }
+
+type DiskBackupImportInput struct {
+	Name string `json:"name"`
+
+	GenerateName string `json:"generate_name"`
+
+	apis.ProjectizedResourceCreateInput
+
+	DiskBackupExportInfo
+
+	BackupStorageId string `json:"backup_storage_id"`
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -1289,9 +1289,9 @@ func _doCreateItem(
 	// 若manager用于name字段，确保name唯一
 	if manager.HasName() {
 		// run name validation after validate create data
-		uniqValues := manager.FetchUniqValues(ctx, dataDict)
 		name, _ := dataDict.GetString("name")
 		if len(name) > 0 {
+			uniqValues := manager.FetchUniqValues(ctx, dataDict)
 			err = NewNameValidator(ctx, manager, ownerId, name, uniqValues)
 			if err != nil {
 				return nil, err

pkg/compute/models/backup_storage.go

@@ -28,6 +28,9 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage"
+	_ "yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/nfs"
+	_ "yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage/object"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
@@ -372,3 +375,16 @@ func (bs *SBackupStorage) Delete(ctx context.Context, userCred mcclient.TokenCre
 	}
 	return bs.SEnabledStatusInfrasResourceBase.Delete(ctx, userCred)
 }
+
+func (bs *SBackupStorage) GetIBackupStorage() (backupstorage.IBackupStorage, error) {
+	accessInfo, err := bs.GetAccessInfo()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetAccessInfo")
+	}
+	log.Infof("GetIBackupStorage %s %s", bs.Id, accessInfo.String())
+	ibs, err := backupstorage.GetBackupStorage(bs.Id, jsonutils.Marshal(accessInfo).(*jsonutils.JSONDict))
+	if err != nil {
+		return nil, errors.Wrap(err, "GetBackupStorage")
+	}
+	return ibs, nil
+}

pkg/compute/models/diskbackups.go

@@ -17,6 +17,7 @@ package models
 import (
 	"context"
 	"database/sql"
+	"net/http"
 	"reflect"
 	"strings"
 	"time"
@@ -27,12 +28,15 @@ import (
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
@@ -136,11 +140,11 @@ func (dm *SDiskBackupManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 	return q, nil
 }
 
-func (self *SDiskBackup) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
-	if self.Status == api.BACKUP_STATUS_DELETING {
-		return httperrors.NewBadRequestError("Cannot delete disk backup in status %s", self.Status)
+func (diskBackup *SDiskBackup) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
+	if diskBackup.Status == api.BACKUP_STATUS_DELETING {
+		return httperrors.NewBadRequestError("Cannot delete disk backup in status %s", diskBackup.Status)
 	}
-	is, err := InstanceBackupJointManager.IsSubBackup(self.Id)
+	is, err := InstanceBackupJointManager.IsSubBackup(diskBackup.Id)
 	if err != nil {
 		return err
 	}
@@ -199,6 +203,9 @@ func (db *SDiskBackup) GetDisk() (*SDisk, error) {
 }
 
 func (db *SDiskBackup) GetStorage() (*SStorage, error) {
+	if len(db.StorageId) == 0 {
+		return nil, errors.Wrap(errors.ErrEmpty, "empty storage_id")
+	}
 	iStorage, err := StorageManager.FetchById(db.StorageId)
 	if err != nil {
 		return nil, err
@@ -446,27 +453,27 @@ func (manager *SDiskBackupManager) OrderByExtraFields(ctx context.Context, q *sq
 	return q, nil
 }
 
-func (self *SDiskBackup) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+func (diskBackup *SDiskBackup) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	return nil
 }
 
-func (self *SDiskBackup) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
-	return db.DeleteModel(ctx, userCred, self)
+func (diskBackup *SDiskBackup) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return db.DeleteModel(ctx, userCred, diskBackup)
 }
 
-func (self *SDiskBackup) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+func (diskBackup *SDiskBackup) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	forceDelete := jsonutils.QueryBoolean(query, "force", false)
-	return self.StartBackupDeleteTask(ctx, userCred, "", forceDelete)
+	return diskBackup.StartBackupDeleteTask(ctx, userCred, "", forceDelete)
 }
 
-func (self *SDiskBackup) StartBackupDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, forceDelete bool) error {
-	self.SetStatus(ctx, userCred, api.BACKUP_STATUS_DELETING, "")
-	log.Infof("start to delete diskbackup %s and set deleting", self.GetId())
+func (diskBackup *SDiskBackup) StartBackupDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, forceDelete bool) error {
+	diskBackup.SetStatus(ctx, userCred, api.BACKUP_STATUS_DELETING, "")
+	log.Infof("start to delete diskbackup %s and set deleting", diskBackup.GetId())
 	params := jsonutils.NewDict()
 	if forceDelete {
 		params.Set("force_delete", jsonutils.JSONTrue)
 	}
-	task, err := taskman.TaskManager.NewTask(ctx, "DiskBackupDeleteTask", self, userCred, params, parentTaskId, "", nil)
+	task, err := taskman.TaskManager.NewTask(ctx, "DiskBackupDeleteTask", diskBackup, userCred, params, parentTaskId, "", nil)
 	if err != nil {
 		return err
 	} else {
@@ -475,21 +482,21 @@ func (self *SDiskBackup) StartBackupDeleteTask(ctx context.Context, userCred mcc
 	return nil
 }
 
-func (self *SDiskBackup) PerformRecovery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupRecoveryInput) (jsonutils.JSONObject, error) {
-	if self.Status != api.BACKUP_STATUS_READY {
-		return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "cannot recover backup in status %s", self.Status)
+func (diskBackup *SDiskBackup) PerformRecovery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupRecoveryInput) (jsonutils.JSONObject, error) {
+	if diskBackup.Status != api.BACKUP_STATUS_READY {
+		return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "cannot recover backup in status %s", diskBackup.Status)
 	}
-	return nil, self.StartRecoveryTask(ctx, userCred, "", input.Name)
+	return nil, diskBackup.StartRecoveryTask(ctx, userCred, "", input.Name)
 }
 
-func (self *SDiskBackup) StartRecoveryTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, diskName string) error {
-	self.SetStatus(ctx, userCred, api.BACKUP_STATUS_RECOVERY, "")
+func (diskBackup *SDiskBackup) StartRecoveryTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, diskName string) error {
+	diskBackup.SetStatus(ctx, userCred, api.BACKUP_STATUS_RECOVERY, "")
 	var params *jsonutils.JSONDict
 	if diskName != "" {
 		params = jsonutils.NewDict()
 		params.Set("disk_name", jsonutils.NewString(diskName))
 	}
-	task, err := taskman.TaskManager.NewTask(ctx, "DiskBackupRecoveryTask", self, userCred, params, parentTaskId, "", nil)
+	task, err := taskman.TaskManager.NewTask(ctx, "DiskBackupRecoveryTask", diskBackup, userCred, params, parentTaskId, "", nil)
 	if err != nil {
 		return err
 	} else {
@@ -536,9 +543,9 @@ func (manager *SDiskBackupManager) CreateBackup(ctx context.Context, owner mccli
 	return backup, nil
 }
 
-func (self *SDiskBackup) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupSyncstatusInput) (jsonutils.JSONObject, error) {
+func (diskBackup *SDiskBackup) PerformSyncstatus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.DiskBackupSyncstatusInput) (jsonutils.JSONObject, error) {
 	var openTask = true
-	count, err := taskman.TaskManager.QueryTasksOfObject(self, time.Now().Add(-3*time.Minute), &openTask).CountWithError()
+	count, err := taskman.TaskManager.QueryTasksOfObject(diskBackup, time.Now().Add(-3*time.Minute), &openTask).CountWithError()
 	if err != nil {
 		return nil, err
 	}
@@ -546,25 +553,25 @@ func (self *SDiskBackup) PerformSyncstatus(ctx context.Context, userCred mcclien
 		return nil, httperrors.NewBadRequestError("Backup has %d task active, can't sync status", count)
 	}
 
-	return nil, StartResourceSyncStatusTask(ctx, userCred, self, "DiskBackupSyncstatusTask", "")
+	return nil, StartResourceSyncStatusTask(ctx, userCred, diskBackup, "DiskBackupSyncstatusTask", "")
 }
 
-func (self *SDiskBackup) PackMetadata() *api.DiskBackupPackMetadata {
+func (diskBackup *SDiskBackup) PackMetadata() *api.DiskBackupPackMetadata {
 	return &api.DiskBackupPackMetadata{
-		OsArch:     self.OsArch,
-		SizeMb:     self.SizeMb,
-		DiskSizeMb: self.DiskSizeMb,
-		DiskType:   self.DiskType,
+		OsArch:     diskBackup.OsArch,
+		SizeMb:     diskBackup.SizeMb,
+		DiskSizeMb: diskBackup.DiskSizeMb,
+		DiskType:   diskBackup.DiskType,
 		// 操作系统类型
-		OsType: self.OsType,
+		OsType: diskBackup.OsType,
 		DiskConfig: &api.SBackupDiskConfig{
-			DiskConfig: self.DiskConfig.DiskConfig,
-			Name:       self.DiskConfig.Name,
+			DiskConfig: diskBackup.DiskConfig.DiskConfig,
+			Name:       diskBackup.DiskConfig.Name,
 		},
 	}
 }
 
-func (manager *SDiskBackupManager) CreateFromPackMetadata(ctx context.Context, owner mcclient.TokenCredential, backupStorageId, id, name string, metadata *api.DiskBackupPackMetadata) (*SDiskBackup, error) {
+func (manager *SDiskBackupManager) CreateFromPackMetadata(ctx context.Context, owner mcclient.IIdentityProvider, backupStorageId, id, name string, metadata *api.DiskBackupPackMetadata) (*SDiskBackup, error) {
 	backup := &SDiskBackup{}
 	backup.SetModelManager(manager, backup)
 	backup.ProjectId = owner.GetProjectId()
@@ -579,7 +586,7 @@ func (manager *SDiskBackupManager) CreateFromPackMetadata(ctx context.Context, o
 	backup.OsArch = metadata.OsArch
 	backup.DiskType = metadata.DiskType
 	backup.OsType = metadata.OsType
-	backup.CloudregionId = "default"
+	backup.CloudregionId = api.DEFAULT_REGION_ID
 	backup.BackupStorageId = backupStorageId
 	backup.Name = name
 	backup.Id = id
@@ -590,3 +597,157 @@ func (manager *SDiskBackupManager) CreateFromPackMetadata(ctx context.Context, o
 	}
 	return backup, nil
 }
+
+type SDiskBackupTotalCount struct {
+	apis.TotalCountBase
+	SizeMb int64
+}
+
+func (manager *SDiskBackupManager) CustomizedTotalCount(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, totalQ *sqlchemy.SQuery) (int, jsonutils.JSONObject, error) {
+	results := SDiskBackupTotalCount{}
+
+	totalQ = totalQ.AppendField(sqlchemy.SUM("size_mb", totalQ.Field("size_mb")))
+
+	err := totalQ.First(&results)
+	if err != nil {
+		return -1, nil, errors.Wrap(err, "SGuestManager query total")
+	}
+
+	return results.Count, jsonutils.Marshal(results).(*jsonutils.JSONDict), nil
+}
+
+func (diskBackup *SDiskBackup) GetDetailsExportInfo(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*api.DiskBackupExportInfo, error) {
+	exportInfo := &api.DiskBackupExportInfo{}
+	metadata := *diskBackup.PackMetadata()
+
+	exportInfo.DiskBackupPackMetadata = metadata
+
+	backupStorage, err := diskBackup.GetBackupStorage()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetBackupStorage")
+	}
+	ibs, err := backupStorage.GetIBackupStorage()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetIBackupStorage")
+	}
+
+	exportInfo.AccessUrl, err = ibs.GetExternalAccessUrl(diskBackup.Id)
+	if err != nil {
+		log.Errorf("SDiskBackup %s(%s) GetExternalAccessUrl fail: %v", diskBackup.GetName(), diskBackup.GetId(), err)
+	}
+
+	return exportInfo, nil
+}
+
+func (manager *SDiskBackupManager) PerformImport(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input api.DiskBackupImportInput,
+) (jsonutils.JSONObject, error) {
+	if len(input.AccessUrl) == 0 {
+		return nil, httperrors.NewInputParameterError("access_url is required")
+	}
+	if len(input.BackupStorageId) == 0 {
+		return nil, httperrors.NewInputParameterError("backup_storage_id is required")
+	}
+
+	backupStorageObj, err := BackupStorageManager.FetchById(input.BackupStorageId)
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2(BackupStorageManager.Keyword(), input.BackupStorageId)
+		}
+		return nil, errors.Wrap(err, "BackupStorageManager.FetchById")
+	}
+	backupStorage := backupStorageObj.(*SBackupStorage)
+	if backupStorage.Status != api.BACKUPSTORAGE_STATUS_ONLINE {
+		return nil, httperrors.NewForbiddenError("can't import disk backup from backup storage with status %s", backupStorage.Status)
+	}
+
+	ownerId, err := manager.FetchOwnerId(ctx, jsonutils.Marshal(input))
+	if err != nil {
+		return nil, errors.Wrap(err, "FetchOwnerId")
+	}
+	if ownerId == nil {
+		ownerId = userCred
+	}
+
+	if len(input.GenerateName) > 0 && manager.EnableGenerateName() {
+		lockman.LockRawObject(ctx, manager.Keyword(), "name")
+		defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name")
+
+		// if enable generateName, alway generate name
+		newName, err := db.GenerateName2(ctx, manager, ownerId, input.GenerateName, nil, 1)
+		if err != nil {
+			return nil, errors.Wrap(err, "GenerateName2")
+		}
+		input.Name = newName
+	} else if len(input.Name) == 0 {
+		return nil, httperrors.NewInputParameterError("name or generate_name is required")
+	}
+
+	uniqValues := manager.FetchUniqValues(ctx, jsonutils.Marshal(input))
+	if err := db.NewNameValidator(ctx, manager, ownerId, input.Name, uniqValues); err != nil {
+		return nil, errors.Wrap(err, "NewNameValidator")
+	}
+
+	diskBackup, err := manager.CreateFromPackMetadata(ctx, ownerId, input.BackupStorageId, "", input.Name, &input.DiskBackupPackMetadata)
+	if err != nil {
+		return nil, errors.Wrap(err, "CreateFromPackMetadata")
+	}
+
+	{
+		notes := diskBackup.GetShortDesc(ctx)
+		db.OpsLog.LogEvent(diskBackup, db.ACT_CREATE, notes, userCred)
+		logclient.AddActionLogWithContext(ctx, diskBackup, logclient.ACT_CREATE, notes, userCred, true)
+	}
+
+	{
+		err := diskBackup.StartImportTask(ctx, userCred, input.DiskBackupImportTaskInput, "")
+		if err != nil {
+			return nil, errors.Wrap(err, "StartImportTask")
+		}
+	}
+
+	details, err := db.GetItemDetails(manager, diskBackup, ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "db.GetItemDetails")
+	}
+
+	return details, nil
+}
+
+func (diskBackup *SDiskBackup) StartImportTask(ctx context.Context, userCred mcclient.TokenCredential, input api.DiskBackupImportTaskInput, parentTaskId string) error {
+	diskBackup.SetStatus(ctx, userCred, api.BACKUP_STATUS_START_IMPORT, "")
+	task, err := taskman.TaskManager.NewTask(ctx, "DiskBackupImportTask", diskBackup, userCred, jsonutils.Marshal(input).(*jsonutils.JSONDict), parentTaskId, "", nil)
+	if err != nil {
+		return errors.Wrap(err, "taskman.TaskManager.NewTask")
+	} else {
+		task.ScheduleRun(nil)
+	}
+	return nil
+}
+
+func (diskBackup *SDiskBackup) DoImport(ctx context.Context, userCred mcclient.TokenCredential, input api.DiskBackupImportTaskInput) error {
+	backupStorage, err := diskBackup.GetBackupStorage()
+	if err != nil {
+		return errors.Wrap(err, "GetBackupStorage")
+	}
+	ibs, err := backupStorage.GetIBackupStorage()
+	if err != nil {
+		return errors.Wrap(err, "GetIBackupStorage")
+	}
+
+	resp, err := http.Get(input.AccessUrl)
+	if err != nil {
+		return errors.Wrap(err, "http.Get")
+	}
+	defer resp.Body.Close()
+
+	err = ibs.SaveBackupFrom(ctx, resp.Body, resp.ContentLength, diskBackup.Id)
+	if err != nil {
+		return errors.Wrap(err, "SaveBackupFrom")
+	}
+
+	return nil
+}

pkg/compute/models/hosts.go

@@ -1871,36 +1871,43 @@ func (hh *SHostManager) GetEnabledKvmHostForBackupStorage(bs *SBackupStorage) (*
 }
 
 func (hh *SHostManager) GetEnabledKvmHostForDiskBackup(backup *SDiskBackup) (*SHost, error) {
-	bs, err := backup.GetBackupStorage()
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to get backupStorage")
-	}
-	storage, err := backup.GetStorage()
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to get storage of diskbackup")
-	}
-
-	hbs, err := HostBackupstorageManager.GetBackupStoragesByBackup(bs.Id)
-	if err != nil {
-		return nil, errors.Wrap(err, "HostBackupstorageManager.GetBackupStoragesByBackup")
-	}
 	hbsCandidates := stringutils2.NewSortedStrings(nil)
-	for i := range hbs {
-		hbsCandidates = hbsCandidates.Append(hbs[i].HostId)
-	}
-	hss, err := HoststorageManager.GetHostStoragesByStorageId(storage.Id)
-	if err != nil {
-		return nil, errors.Wrap(err, "HoststorageManager.GetStorages")
-	}
 	hssCandidates := stringutils2.NewSortedStrings(nil)
-	for i := range hss {
-		hssCandidates = hssCandidates.Append(hss[i].HostId)
-	}
 	var candidates []string
-	if len(hbsCandidates) == 0 {
-		candidates = []string(hssCandidates)
+
+	{
+		bs, err := backup.GetBackupStorage()
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to get backupStorage")
+		}
+		hbs, err := HostBackupstorageManager.GetBackupStoragesByBackup(bs.Id)
+		if err != nil {
+			return nil, errors.Wrap(err, "HostBackupstorageManager.GetBackupStoragesByBackup")
+		}
+
+		for i := range hbs {
+			hbsCandidates = hbsCandidates.Append(hbs[i].HostId)
+		}
+	}
+	if len(backup.StorageId) > 0 {
+		storage, err := backup.GetStorage()
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to get storage of diskbackup")
+		}
+		hss, err := HoststorageManager.GetHostStoragesByStorageId(storage.Id)
+		if err != nil {
+			return nil, errors.Wrap(err, "HoststorageManager.GetStorages")
+		}
+		for i := range hss {
+			hssCandidates = hssCandidates.Append(hss[i].HostId)
+		}
+		if len(hbsCandidates) == 0 {
+			candidates = []string(hssCandidates)
+		} else {
+			candidates = []string(stringutils2.Intersect(hbsCandidates, hssCandidates))
+		}
 	} else {
-		candidates = []string(stringutils2.Intersect(hbsCandidates, hssCandidates))
+		candidates = []string(hbsCandidates)
 	}
 
 	host, err := HostManager.GetEnabledKvmHost(candidates)
@@ -1914,6 +1921,7 @@ func (hh *SHostManager) GetEnabledKvmHost(candidates []string) (*SHost, error) {
 	hostq := HostManager.Query().IsTrue("enabled")
 	hostq = hostq.Equals("host_status", api.HOST_ONLINE)
 	hostq = hostq.In("host_type", []string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_KVM, api.HOST_TYPE_CONTAINER})
+	hostq = hostq.IsNullOrEmpty("manager_id")
 	if len(candidates) > 0 {
 		hostq = hostq.In("id", candidates)
 	}

pkg/compute/service/service.go

@@ -196,7 +196,7 @@ func StartServiceWithJobsAndApp(jobs func(cron *cronman.SCronJobManager), appCll
 		cron.AddJobEveryFewHour("AutoPurgeSplitable", 4, 30, 0, db.AutoPurgeSplitable, false)
 
 		cron.AddJobEveryFewHour("AutoDiskSnapshot", 1, 5, 0, models.DiskManager.AutoDiskSnapshot, false)
-		cron.AddJobEveryFewHour("AutoServerSnapshot", 1, 5, 0, models.InstanceSnapshotManager.AutoServerSnapshot, false)
+		cron.AddJobEveryFewHour("AutoServerSnapshot", 1, 10, 0, models.InstanceSnapshotManager.AutoServerSnapshot, false)
 		cron.AddJobEveryFewHour("SnapshotsCleanup", 1, 35, 0, models.SnapshotManager.CleanupSnapshots, false)
 		cron.AddJobEveryFewHour("InstanceSnapshotsCleanup", 1, 35, 0, models.InstanceSnapshotManager.CleanupInstanceSnapshots, false)
 

pkg/compute/tasks/backup/disk_backup_import_task.go

@@ -0,0 +1,83 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type DiskBackupImportTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(DiskBackupImportTask{})
+}
+
+func (importTask *DiskBackupImportTask) taskFailed(ctx context.Context, backup *models.SDiskBackup, reason jsonutils.JSONObject, status string) {
+	reasonStr, _ := reason.GetString()
+	backup.SetStatus(ctx, importTask.UserCred, status, reasonStr)
+	logclient.AddActionLogWithStartable(importTask, backup, logclient.ACT_CREATE, reason, importTask.UserCred, false)
+	importTask.SetStageFailed(ctx, reason)
+}
+
+func (importTask *DiskBackupImportTask) taksSuccess(ctx context.Context, backup *models.SDiskBackup, data *jsonutils.JSONDict) {
+	backup.SetStatus(ctx, importTask.UserCred, api.BACKUP_STATUS_READY, "")
+	logclient.AddActionLogWithStartable(importTask, backup, logclient.ACT_CREATE, backup.GetShortDesc(ctx), importTask.UserCred, true)
+	notifyclient.EventNotify(ctx, importTask.UserCred, notifyclient.SEventNotifyParam{
+		Obj:    backup,
+		Action: notifyclient.ActionCreate,
+	})
+	importTask.SetStageComplete(ctx, data)
+}
+
+func (importTask *DiskBackupImportTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	backup := obj.(*models.SDiskBackup)
+	input := api.DiskBackupImportTaskInput{}
+	err := importTask.Params.Unmarshal(&input)
+	if err != nil {
+		importTask.taskFailed(ctx, backup, jsonutils.NewString(err.Error()), api.BACKUP_STATUS_IMPORT_FAILED)
+		return
+	}
+
+	backup.SetStatus(ctx, importTask.UserCred, api.BACKUP_STATUS_IMPORTING, "")
+
+	importTask.SetStage("OnImportComplete", nil)
+	taskman.LocalTaskRun(importTask, func() (jsonutils.JSONObject, error) {
+		err := backup.DoImport(ctx, importTask.UserCred, input)
+		if err != nil {
+			return nil, errors.Wrap(err, "DoImport")
+		}
+		return nil, nil
+	})
+}
+
+func (importTask *DiskBackupImportTask) OnImportComplete(ctx context.Context, backup *models.SDiskBackup, data jsonutils.JSONObject) {
+	importTask.taksSuccess(ctx, backup, nil)
+}
+
+func (importTask *DiskBackupImportTask) OnImportCompleteFailed(ctx context.Context, backup *models.SDiskBackup, data jsonutils.JSONObject) {
+	importTask.taskFailed(ctx, backup, data, api.BACKUP_STATUS_IMPORT_FAILED)
+}

pkg/hostman/storageman/backup.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path"
 	"strings"
 
@@ -115,7 +116,7 @@ func doBackupDisk(ctx context.Context, snapshotPath string, diskBackup *SDiskBac
 		return 0, errors.Wrap(err, "GetBackupStorage")
 	}
 
-	err = backupStorage.SaveBackupFrom(ctx, backupPath, diskBackup.BackupId)
+	err = backupstorage.SaveBackupFromFile(ctx, backupPath, diskBackup.BackupId, backupStorage)
 	if err != nil {
 		return 0, errors.Wrap(err, "SaveBackupFrom")
 	}
@@ -269,7 +270,7 @@ func DoInstancePackBackup(ctx context.Context, backupInfo SStoragePackInstanceBa
 	{
 		// save snapshot metadata
 		packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
-		err = ioutil.WriteFile(packageMetadataPath, []byte(jsonutils.Marshal(backupInfo.Metadata).PrettyString()), 0644)
+		err = os.WriteFile(packageMetadataPath, []byte(jsonutils.Marshal(backupInfo.Metadata).PrettyString()), 0644)
 		if err != nil {
 			return "", errors.Wrapf(err, "unable to write to %s", packageMetadataPath)
 		}
@@ -299,7 +300,7 @@ func DoInstancePackBackup(ctx context.Context, backupInfo SStoragePackInstanceBa
 		if exists {
 			tried++
 		} else {
-			err := backupStorage.SaveBackupInstanceFrom(ctx, tmpPkgFilename, finalPackageFileName)
+			err := backupstorage.SaveBackupInstanceFromFile(ctx, tmpPkgFilename, finalPackageFileName, backupStorage)
 			if err != nil {
 				return "", errors.Wrap(err, "SaveBackupInstanceFrom")
 			}
@@ -353,7 +354,7 @@ func DoInstanceUnpackBackup(ctx context.Context, backupInfo SStorageUnpackInstan
 
 	// unpack metadata
 	packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
-	metadataBytes, err := ioutil.ReadFile(packageMetadataPath)
+	metadataBytes, err := os.ReadFile(packageMetadataPath)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "unable to read metadata file")
 	}
@@ -374,7 +375,7 @@ func DoInstanceUnpackBackup(ctx context.Context, backupInfo SStorageUnpackInstan
 			backupId := db.DefaultUUIDGenerator()
 			backupIds[i] = backupId
 			packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i))
-			err := backupStorage.SaveBackupFrom(ctx, packageDiskPath, backupId)
+			err := backupstorage.SaveBackupFromFile(ctx, packageDiskPath, backupId, backupStorage)
 			if err != nil {
 				return nil, nil, errors.Wrapf(err, "SaveBackupFrom %s %s", packageDiskPath, backupId)
 			}

pkg/hostman/storageman/backupstorage/backup_storage.go

@@ -16,6 +16,8 @@ package backupstorage
 
 import (
 	"context"
+	"io"
+	"os"
 	"sync"
 
 	"yunion.io/x/jsonutils"
@@ -28,7 +30,7 @@ type IBackupStorageFactory interface {
 
 type IBackupStorage interface {
 	// 从指定路径拷贝磁盘文件到备份存储
-	SaveBackupFrom(ctx context.Context, srcFilename string, bakcupId string) error
+	SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupId string) error
 	// 将备份backupId对应的备份文件拷贝到指定的文件路径
 	RestoreBackupTo(ctx context.Context, targetFilename string, backupId string) error
 	// 删除备份
@@ -37,7 +39,7 @@ type IBackupStorage interface {
 	IsBackupExists(backupId string) (bool, error)
 
 	// 从指定路径拷贝主机备份文件到备份存储
-	SaveBackupInstanceFrom(ctx context.Context, srcFilename string, bakcupInstanceId string) error
+	SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupInstanceId string) error
 	// 将备份backupId对应的备份文件拷贝到指定的文件路径
 	RestoreBackupInstanceTo(ctx context.Context, targetFilename string, backupInstanceId string) error
 	// 删除备份
@@ -52,6 +54,9 @@ type IBackupStorage interface {
 
 	// 存储是否在线
 	IsOnline() (bool, string, error)
+
+	// 获取外部访问地址
+	GetExternalAccessUrl(backupId string) (string, error)
 }
 
 var factories []IBackupStorageFactory
@@ -95,3 +100,30 @@ func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils
 		return ibs, nil
 	}
 }
+
+func SaveBackupFromFile(ctx context.Context, srcFilename string, bakcupId string, storage IBackupStorage) error {
+	fileInfo, err := os.Stat(srcFilename)
+	if err != nil {
+		return errors.Wrapf(err, "stat %s", srcFilename)
+	}
+	file, err := os.Open(srcFilename)
+	if err != nil {
+		return errors.Wrapf(err, "Open %s", srcFilename)
+	}
+	defer file.Close()
+
+	return storage.SaveBackupFrom(ctx, file, fileInfo.Size(), bakcupId)
+}
+
+func SaveBackupInstanceFromFile(ctx context.Context, srcFilename string, bakcupInstanceId string, storage IBackupStorage) error {
+	fileInfo, err := os.Stat(srcFilename)
+	if err != nil {
+		return errors.Wrapf(err, "stat %s", srcFilename)
+	}
+	file, err := os.Open(srcFilename)
+	if err != nil {
+		return errors.Wrapf(err, "Open %s", srcFilename)
+	}
+	defer file.Close()
+	return storage.SaveBackupInstanceFrom(ctx, file, fileInfo.Size(), bakcupInstanceId)
+}

pkg/hostman/storageman/backupstorage/backup_storage_nfs.go

@@ -1,527 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package backupstorage
-
-import (
-	"context"
-	"fmt"
-	"io/ioutil"
-	"path"
-	"strings"
-	"sync"
-	"time"
-
-	"yunion.io/x/jsonutils"
-	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/util/qemuimgfmt"
-
-	api "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
-	"yunion.io/x/onecloud/pkg/hostman/options"
-	"yunion.io/x/onecloud/pkg/util/fileutils2"
-	"yunion.io/x/onecloud/pkg/util/procutils"
-	"yunion.io/x/onecloud/pkg/util/qemuimg"
-)
-
-var ErrorBackupStorageOffline error = errors.Error(api.BackupStorageOffline)
-
-type SNFSBackupStorage struct {
-	BackupStorageId string
-	Path            string
-	NfsHost         string
-	NfsSharedDir    string
-	lock            *sync.Mutex
-	userNumber      int
-}
-
-func NewNFSBackupStorage(backupStorageId, nfsHost, nfsSharedDir string) *SNFSBackupStorage {
-	return &SNFSBackupStorage{
-		BackupStorageId: backupStorageId,
-		NfsHost:         nfsHost,
-		NfsSharedDir:    nfsSharedDir,
-		Path:            path.Join(options.HostOptions.LocalBackupStoragePath, backupStorageId),
-		lock:            &sync.Mutex{},
-	}
-}
-
-func (s *SNFSBackupStorage) getBackupDir() string {
-	return path.Join(s.Path, "backups")
-}
-
-func (s *SNFSBackupStorage) getPackageDir() string {
-	return path.Join(s.Path, "backuppacks")
-}
-
-func (s *SNFSBackupStorage) checkAndMount() error {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	if !fileutils2.Exists(s.Path) {
-		output, err := procutils.NewCommand("mkdir", "-p", s.Path).Output()
-		if err != nil {
-			log.Errorf("mkdir %s failed: %s", s.Path, output)
-			return errors.Wrapf(err, "mkdir %s failed: %s", s.Path, output)
-		}
-	}
-	if err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", s.Path).Run(); err == nil {
-		s.userNumber++
-		return nil
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx,
-		"mount", "-t", "nfs", fmt.Sprintf("%s:%s", s.NfsHost, s.NfsSharedDir), s.Path).Run()
-	if err != nil {
-		return errors.Wrap(ErrorBackupStorageOffline, err.Error())
-	}
-	backupDir := s.getBackupDir()
-	if !fileutils2.Exists(backupDir) {
-		output, err := procutils.NewCommand("mkdir", "-p", backupDir).Output()
-		if err != nil {
-			log.Errorf("mkdir %s failed: %s", backupDir, output)
-			return errors.Wrapf(err, "mkdir %s failed: %s", backupDir, output)
-		}
-	}
-	packageDir := s.getPackageDir()
-	if !fileutils2.Exists(packageDir) {
-		output, err := procutils.NewCommand("mkdir", "-p", packageDir).Output()
-		if err != nil {
-			log.Errorf("mkdir %s failed: %s", packageDir, output)
-			return errors.Wrapf(err, "mkdir %s failed: %s", packageDir, output)
-		}
-	}
-	s.userNumber++
-	return nil
-}
-
-func (s *SNFSBackupStorage) unMount() error {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	s.userNumber--
-	if s.userNumber > 0 {
-		return nil
-	}
-	out, err := procutils.NewRemoteCommandAsFarAsPossible("umount", s.Path).Output()
-	if err != nil {
-		return errors.Wrapf(err, "umount %s failed %s", s.Path, out)
-	}
-	return nil
-}
-
-func (s *SNFSBackupStorage) CopyBackupFrom(srcFilename string, backupId string) error {
-	err := s.checkAndMount()
-	if err != nil {
-		return errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	backupDir := s.getBackupDir()
-	targetFilename := path.Join(backupDir, backupId)
-	if output, err := procutils.NewCommand("cp", srcFilename, targetFilename).Output(); err != nil {
-		log.Errorf("unable to cp %s to %s: %s", srcFilename, targetFilename, output)
-		return errors.Wrapf(err, "cp %s to %s failed and output is %q", srcFilename, targetFilename, output)
-	}
-	return nil
-}
-
-func (s *SNFSBackupStorage) CopyBackupTo(targetFilename string, backupId string) error {
-	err := s.checkAndMount()
-	if err != nil {
-		return errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	backupDir := s.getBackupDir()
-	srcFilename := path.Join(backupDir, backupId)
-	if output, err := procutils.NewCommand("cp", srcFilename, targetFilename).Output(); err != nil {
-		log.Errorf("unable to cp %s to %s: %s", srcFilename, targetFilename, output)
-		return errors.Wrapf(err, "cp %s to %s failed and output is %q", srcFilename, targetFilename, output)
-	}
-	return nil
-}
-
-const (
-	PackageDiskFilename     = "disk"
-	PackageMetadataFilename = "metadata"
-)
-
-/*
-func (s *SNFSBackupStorage) Pack(backupId string, packageName string, metadata jsonutils.JSONObject) error {
-	err := s.checkAndMount()
-	if err != nil {
-		return errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	lockman.LockRawObject(context.Background(), "package", packageName)
-	defer lockman.ReleaseRawObject(context.Background(), "package", packageName)
-	backupDir := s.getBackupDir()
-	backupPath := path.Join(backupDir, backupId)
-	packageDir := s.getPackageDir()
-	packagePath := path.Join(packageDir, packageName)
-	packageFilename := path.Join(packageDir, packageName+".tar")
-	if fileutils2.Exists(packageFilename) {
-		return errors.Error("A package with the same name already exists")
-	}
-	if fileutils2.Exists(packagePath) {
-		// delete residual data
-		if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
-			log.Errorf("unable to rm %s: %s", packagePath, output)
-			return errors.Wrapf(err, "rm %s failed and output is %q", packagePath, output)
-		}
-	}
-	output, err := procutils.NewCommand("mkdir", "-p", packagePath).Output()
-	if err != nil {
-		log.Errorf("mkdir %s failed: %s", packagePath, output)
-		return errors.Wrapf(err, "mkdir %s failed: %s", packageDir, output)
-	}
-	defer func() {
-		if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
-			log.Errorf("unable to rm %s: %s", packagePath, output)
-		}
-	}()
-	packageDiskPath := path.Join(packagePath, PackageDiskFilename)
-	if output, err := procutils.NewCommand("cp", backupPath, packageDiskPath).Output(); err != nil {
-		log.Errorf("unable to cp %s to %s: %s", backupPath, packageDiskPath, output)
-		return errors.Wrapf(err, "cp %s to %s failed and output is %q", backupPath, packageDiskPath, output)
-	}
-	packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
-	err = ioutil.WriteFile(packageMetadataPath, []byte(metadata.PrettyString()), 0644)
-	if err != nil {
-		return errors.Wrapf(err, "unable to write to %s", packageMetadataPath)
-	}
-	// tar
-	if output, err := procutils.NewCommand("tar", "-cf", packageFilename, packagePath).Output(); err != nil {
-		log.Errorf("unable to 'tar -cf %s %s': %s", packageFilename, packagePath, output)
-		return errors.Wrap(err, "unable to tar")
-	}
-	return nil
-}
-
-func (s *SNFSBackupStorage) UnPack(backupId string, packageName string) (jsonutils.JSONObject, error) {
-	err := s.checkAndMount()
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	lockman.LockRawObject(context.Background(), "package", packageName)
-	defer lockman.ReleaseRawObject(context.Background(), "package", packageName)
-	backupDir := s.getBackupDir()
-	backupPath := path.Join(backupDir, backupId)
-	packageDir := s.getPackageDir()
-	packagePath := path.Join(packageDir, packageName)
-	packageFilename := path.Join(packageDir, packageName+".tar")
-	if !fileutils2.Exists(packageFilename) {
-		return nil, errors.Wrapf(err, "package %s does not exists", packageName)
-	}
-	if fileutils2.Exists(packagePath) {
-		// delete residual data
-		if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
-			log.Errorf("unable to rm %s: %s", packagePath, output)
-			return nil, errors.Wrapf(err, "rm %s failed and output is %q", packagePath, output)
-		}
-	}
-	// untar
-	if output, err := procutils.NewCommand("tar", "-xf", packageFilename, packagePath).Output(); err != nil {
-		log.Errorf("unable to 'tar -xf %s %s': %s", packageFilename, packagePath, output)
-		return nil, errors.Wrap(err, "unable to untar")
-	}
-	defer func() {
-		if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
-			log.Errorf("unable to rm %s: %s", packagePath, output)
-		}
-	}()
-	packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
-	metadataBytes, err := ioutil.ReadFile(packageMetadataPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to read metadata file")
-	}
-	metadata, err := jsonutils.ParseQueryString(string(metadataBytes))
-	if err != nil {
-		return nil, errors.Wrap(err, "unable to parse string to json")
-	}
-	packageDiskPath := path.Join(packagePath, PackageDiskFilename)
-	if output, err := procutils.NewCommand("mv", packageDiskPath, backupPath).Output(); err != nil {
-		return nil, errors.Wrapf(err, "mv %s to %s failed and output is %q", packageDiskPath, backupPath, output)
-	}
-	return metadata, nil
-}
-*/
-
-func (s *SNFSBackupStorage) InstancePack(ctx context.Context, packageName string, backupIds []string, metadata *api.InstanceBackupPackMetadata) (string, error) {
-	err := s.checkAndMount()
-	if err != nil {
-		return "", errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-
-	tmpFileDir, err := ioutil.TempDir(options.HostOptions.LocalBackupTempPath, "pack")
-	if err != nil {
-		return "", errors.Wrap(err, "create tempdir")
-	}
-	defer func() {
-		if output, err := procutils.NewCommand("rm", "-rf", tmpFileDir).Output(); err != nil {
-			log.Errorf("unable to rm %s: %s", tmpFileDir, output)
-		}
-	}()
-
-	backupDir := s.getBackupDir()
-
-	packagePath := path.Join(tmpFileDir, packageName)
-	tmpPkgFilename := path.Join(tmpFileDir, packageName+".tar")
-
-	err = func() error {
-		if _, err := procutils.NewCommand("touch", packageName).Output(); err != nil {
-			return errors.Wrapf(err, "create %s", packageName)
-		}
-		return nil
-	}()
-	if err != nil {
-		return "", errors.Wrap(err, "A package with the same name already exists")
-	}
-
-	output, err := procutils.NewCommand("mkdir", "-p", packagePath).Output()
-	if err != nil {
-		log.Errorf("mkdir %s failed: %s", packagePath, output)
-		return "", errors.Wrapf(err, "mkdir %s failed: %s", packagePath, output)
-	}
-	defer func() {
-		if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
-			log.Errorf("unable to rm %s: %s", packagePath, output)
-		}
-	}()
-	// copy disk files
-	for i, backupId := range backupIds {
-		packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i))
-		backupPath := path.Join(backupDir, backupId)
-		if output, err := procutils.NewCommand("cp", backupPath, packageDiskPath).Output(); err != nil {
-			log.Errorf("unable to cp %s to %s: %s", backupPath, packageDiskPath, output)
-			return "", errors.Wrapf(err, "cp %s to %s failed and output is %q", backupPath, packageDiskPath, output)
-		}
-	}
-	// save snapshot metadata
-	packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
-	err = ioutil.WriteFile(packageMetadataPath, []byte(jsonutils.Marshal(metadata).PrettyString()), 0644)
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to write to %s", packageMetadataPath)
-	}
-	// tar
-	if output, err := procutils.NewCommand("tar", "-cf", tmpPkgFilename, "-C", tmpFileDir, packageName).Output(); err != nil {
-		log.Errorf("unable to 'tar -cf %s -C %s %s': %s", tmpPkgFilename, tmpFileDir, packageName, output)
-		return "", errors.Wrap(err, "unable to tar")
-	}
-	// move to pack dir
-	var packageFilename string
-	{
-		lockman.LockRawObject(ctx, "package", packageName)
-		defer lockman.ReleaseRawObject(ctx, "package", packageName)
-
-		packageDir := s.getPackageDir()
-
-		// find the filename
-		tried := 0
-		packageFilename = path.Join(packageDir, packageName+".tar")
-		for fileutils2.Exists(packageFilename) {
-			tried++
-			packageFilename = path.Join(packageDir, fmt.Sprintf("%s-%d.tar", packageName, tried))
-		}
-
-		// move file
-		if output, err := procutils.NewCommand("cp", "-a", tmpPkgFilename, packageFilename).Output(); err != nil {
-			log.Errorf("cp %s to %s fail: %s", tmpPkgFilename, packageFilename, output)
-			return "", errors.Wrap(err, "cp")
-		}
-	}
-	return packageFilename, nil
-}
-
-func (s *SNFSBackupStorage) InstanceUnpack(ctx context.Context, packageName string, metadataOnly bool) ([]string, *api.InstanceBackupPackMetadata, error) {
-	err := s.checkAndMount()
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-
-	// create temp working dir
-	tmpFileDir, err := ioutil.TempDir(options.HostOptions.LocalBackupTempPath, "unpack")
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "create tempdir")
-	}
-	defer func() {
-		if output, err := procutils.NewCommand("rm", "-rf", tmpFileDir).Output(); err != nil {
-			log.Errorf("unable to rm %s: %s", tmpFileDir, output)
-		}
-	}()
-
-	backupDir := s.getBackupDir()
-	packageDir := s.getPackageDir()
-	if strings.HasSuffix(packageName, ".tar") {
-		// remove suffix
-		packageName = packageName[:len(packageName)-4]
-	}
-	packageFilename := path.Join(packageDir, packageName+".tar")
-	if !fileutils2.Exists(packageFilename) {
-		return nil, nil, errors.Wrapf(errors.ErrNotFound, "package %s does not exists", packageName)
-	}
-
-	// untar to temp dir
-	packagePath := path.Join(tmpFileDir, packageName)
-	log.Infof("unpack to %s", packagePath)
-	untarArgs := []string{
-		"-xf", packageFilename, "-C", tmpFileDir,
-	}
-	if metadataOnly {
-		untarArgs = append(untarArgs, fmt.Sprintf("%s/metadata", packageName))
-	} else {
-		untarArgs = append(untarArgs, packageName)
-	}
-	if output, err := procutils.NewCommand("tar", untarArgs...).Output(); err != nil {
-		log.Errorf("unable to 'tar -xf %s -C %s %s': %s", packageFilename, tmpFileDir, packageName, output)
-		return nil, nil, errors.Wrap(err, "unable to untar")
-	}
-	// unpack metadata
-	packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
-	metadataBytes, err := ioutil.ReadFile(packageMetadataPath)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "unable to read metadata file")
-	}
-	metadataJson, err := jsonutils.Parse(metadataBytes)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "unable to parse string to json")
-	}
-	metadata := &api.InstanceBackupPackMetadata{}
-	err = metadataJson.Unmarshal(metadata)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "unmarshal backup metadata")
-	}
-	// copy disk files only if !metadataOnly
-	backupIds := make([]string, len(metadata.DiskMetadatas))
-	if !metadataOnly {
-		for i := 0; i < len(metadata.DiskMetadatas); i++ {
-			backupId := db.DefaultUUIDGenerator()
-			backupIds[i] = backupId
-			backupPath := path.Join(backupDir, backupId)
-			packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i))
-			if output, err := procutils.NewCommand("cp", "-a", packageDiskPath, backupPath).Output(); err != nil {
-				return nil, nil, errors.Wrapf(err, "mv %s to %s failed and output is %q", packageDiskPath, backupPath, output)
-			}
-		}
-	}
-	return backupIds, metadata, nil
-}
-
-func (s *SNFSBackupStorage) ConvertFrom(srcPath string, format qemuimgfmt.TImageFormat, backupId string) (int, error) {
-	err := s.checkAndMount()
-	if err != nil {
-		return 0, errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	backupDir := s.getBackupDir()
-	destPath := path.Join(backupDir, backupId)
-	srcInfo := qemuimg.SImageInfo{
-		Path:     srcPath,
-		Format:   format,
-		IoLevel:  qemuimg.IONiceNone,
-		Password: "",
-	}
-	destInfo := qemuimg.SImageInfo{
-		Path:     destPath,
-		Format:   qemuimgfmt.QCOW2,
-		IoLevel:  qemuimg.IONiceNone,
-		Password: "",
-	}
-	err = qemuimg.Convert(srcInfo, destInfo, true, nil)
-	if err != nil {
-		return 0, err
-	}
-	newImage, err := qemuimg.NewQemuImage(destPath)
-	if err != nil {
-		return 0, err
-	}
-	return newImage.GetActualSizeMB(), nil
-}
-
-func (s *SNFSBackupStorage) ConvertTo(destPath string, format qemuimgfmt.TImageFormat, backupId string) error {
-	err := s.checkAndMount()
-	if err != nil {
-		return errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	backupDir := s.getBackupDir()
-	srcPath := path.Join(backupDir, backupId)
-	srcInfo := qemuimg.SImageInfo{
-		Path:     srcPath,
-		Format:   qemuimgfmt.QCOW2,
-		IoLevel:  qemuimg.IONiceNone,
-		Password: "",
-	}
-	destInfo := qemuimg.SImageInfo{
-		Path:     destPath,
-		Format:   format,
-		IoLevel:  qemuimg.IONiceNone,
-		Password: "",
-	}
-	var workerOpts []string
-	if options.HostOptions.RestrictQemuImgConvertWorker {
-		workerOpts = nil
-	} else {
-		workerOpts = []string{"-W", "-m", "16"}
-	}
-	return qemuimg.Convert(srcInfo, destInfo, false, workerOpts)
-}
-
-func (s *SNFSBackupStorage) GetBackupPath(backupId string) string {
-	backupDir := s.getBackupDir()
-	return path.Join(backupDir, backupId)
-}
-
-func (s *SNFSBackupStorage) RemoveBackup(backupId string) error {
-	err := s.checkAndMount()
-	if err != nil {
-		return errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	backupDir := s.getBackupDir()
-	filename := path.Join(backupDir, backupId)
-	if !fileutils2.Exists(filename) {
-		return nil
-	}
-	if output, err := procutils.NewCommand("rm", filename).Output(); err != nil {
-		log.Errorf("unable to rm %s: %s", filename, output)
-		return errors.Wrapf(err, "rm %s failed and output is %q", filename, output)
-	}
-	return nil
-}
-
-func (s *SNFSBackupStorage) IsExists(backupId string) (bool, error) {
-	err := s.checkAndMount()
-	if err != nil {
-		return false, errors.Wrap(err, "unable to checkAndMount")
-	}
-	defer s.unMount()
-	backupDir := s.getBackupDir()
-	filename := path.Join(backupDir, backupId)
-	return fileutils2.Exists(filename), nil
-}
-
-func (s *SNFSBackupStorage) IsOnline() (bool, string, error) {
-	err := s.checkAndMount()
-	if errors.Cause(err) == ErrorBackupStorageOffline {
-		return false, err.Error(), nil
-	}
-	if err != nil {
-		return false, "", err
-	}
-	s.unMount()
-	return true, "", nil
-}

pkg/hostman/storageman/backupstorage/nfs/nfs.go

@@ -17,6 +17,8 @@ package nfs
 import (
 	"context"
 	"fmt"
+	"io"
+	"os"
 	"path"
 	"sync"
 	"time"
@@ -122,15 +124,15 @@ func (s *SNFSBackupStorage) unMount() error {
 	return nil
 }
 
-func (s *SNFSBackupStorage) SaveBackupFrom(ctx context.Context, srcFilename string, backupId string) error {
-	return s.saveFile(ctx, srcFilename, backupId, s.getBackupDiskPath)
+func (s *SNFSBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error {
+	return s.saveFile(ctx, srcFile, fileSize, backupId, s.getBackupDiskPath)
 }
 
-func (s *SNFSBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFilename string, backupId string) error {
-	return s.saveFile(ctx, srcFilename, backupId, s.getBackupDiskPath)
+func (s *SNFSBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error {
+	return s.saveFile(ctx, srcFile, fileSize, backupId, s.getBackupDiskPath)
 }
 
-func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFilename string, id string, getPathFunc func(string) string) error {
+func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFile io.Reader, fileSize int64, id string, getPathFunc func(string) string) error {
 	err := s.checkAndMount()
 	if err != nil {
 		return errors.Wrap(err, "unable to checkAndMount")
@@ -138,9 +140,16 @@ func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFilename string, id
 	defer s.unMount()
 
 	targetFilename := getPathFunc(id)
-	if output, err := procutils.NewCommand("cp", srcFilename, targetFilename).Output(); err != nil {
-		log.Errorf("unable to cp %s to %s: %s", srcFilename, targetFilename, output)
-		return errors.Wrapf(err, "cp %s to %s failed and output is %q", srcFilename, targetFilename, output)
+
+	targetFile, err := os.Create(targetFilename)
+	if err != nil {
+		return errors.Wrap(err, "os.Create")
+	}
+	defer targetFile.Close()
+	_, err = io.CopyN(targetFile, srcFile, fileSize)
+	if err != nil {
+		log.Errorf("unable to io.Copy to %s: %s", targetFilename, err)
+		return errors.Wrap(err, "io.Copy")
 	}
 	return nil
 }
@@ -224,3 +233,7 @@ func (s *SNFSBackupStorage) IsOnline() (bool, string, error) {
 	s.unMount()
 	return true, "", nil
 }
+
+func (s *SNFSBackupStorage) GetExternalAccessUrl(backupId string) (string, error) {
+	return "", errors.ErrNotSupported
+}

pkg/hostman/storageman/backupstorage/object/factory.go

@@ -41,7 +41,14 @@ func (factory *sObjectBackupStorageFactory) NewBackupStore(backupStroageId strin
 	if len(accessInfo.ObjectSecret) == 0 {
 		return nil, errors.Wrap(httperrors.ErrInputParameter, "need object_secret in backup_storage_access_info")
 	}
-	return newObjectBackupStorage(backupStroageId, accessInfo.ObjectBucketUrl, accessInfo.ObjectAccessKey, accessInfo.ObjectSecret, objectstore.S3SignVersion(accessInfo.ObjectSignVer))
+	return newObjectBackupStorage(
+		backupStroageId,
+		accessInfo.ObjectBucketUrl,
+		accessInfo.ObjectAccessKey,
+		accessInfo.ObjectSecret,
+		objectstore.S3SignVersion(accessInfo.ObjectSignVer),
+		accessInfo.ObjectBucketUrlExt,
+	)
 }
 
 func init() {

pkg/hostman/storageman/backupstorage/object/object.go

@@ -17,10 +17,14 @@ package object
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
+	"github.com/coredns/coredns/plugin/pkg/log"
 	"yunion.io/x/cloudmux/pkg/cloudprovider"
 	"yunion.io/x/cloudmux/pkg/multicloud/objectstore"
 	"yunion.io/x/pkg/errors"
@@ -32,10 +36,14 @@ type SObjectBackupStorage struct {
 
 	bucket string
 
+	extBucket string
+
 	store *objectstore.SObjectStoreClient
+
+	extStore *objectstore.SObjectStoreClient
 }
 
-func newObjectBackupStorage(backupStorageId, bucketUrl, accessKey, secret string, signVer objectstore.S3SignVersion) (*SObjectBackupStorage, error) {
+func newObjectBackupStorage(backupStorageId, bucketUrl, accessKey, secret string, signVer objectstore.S3SignVersion, bucketUrlExt string) (*SObjectBackupStorage, error) {
 	bucket, endpoint, err := parseBucketUrl(bucketUrl)
 	if err != nil {
 		return nil, errors.Wrapf(err, "parseBucketUrl %s", bucketUrl)
@@ -49,12 +57,31 @@ func newObjectBackupStorage(backupStorageId, bucketUrl, accessKey, secret string
 		return nil, errors.Wrap(err, "NewObjectStoreClient")
 	}
 
+	var extStore *objectstore.SObjectStoreClient
+	var extBucket string
+	if len(bucketUrlExt) > 0 {
+		b, extEndpoint, err := parseBucketUrl(bucketUrlExt)
+		if err != nil {
+			log.Errorf("parseBucketUrl %s: %v", bucketUrlExt, err)
+		} else {
+			extCfg := objectstore.NewObjectStoreClientConfig(extEndpoint, accessKey, secret)
+			extStore, _ = objectstore.NewObjectStoreClient(extCfg)
+			if extStore != nil {
+				extBucket = b
+			}
+		}
+	}
+
 	return &SObjectBackupStorage{
 		BackupStorageId: backupStorageId,
 
 		bucket: bucket,
 
 		store: store,
+
+		extBucket: extBucket,
+
+		extStore: extStore,
 	}, nil
 }
 
@@ -98,32 +125,34 @@ func (s *SObjectBackupStorage) getBucket() (cloudprovider.ICloudBucket, error) {
 	return bucket, nil
 }
 
-func (s *SObjectBackupStorage) SaveBackupFrom(ctx context.Context, srcFilename string, backupId string) error {
-	return s.saveObject(ctx, srcFilename, backupId, s.getBackupKey)
+func (s *SObjectBackupStorage) getExtBucket() (cloudprovider.ICloudBucket, error) {
+	if s.extStore == nil || len(s.extBucket) == 0 {
+		return nil, errors.Wrap(errors.ErrInvalidStatus, "extStore is nil or extBucket is empty")
+	}
+	bucket, err := s.extStore.GetIRegion().GetIBucketByName(s.extBucket)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetIBucketByName")
+	}
+	return bucket, nil
 }
 
-func (s *SObjectBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFilename string, backupId string) error {
-	return s.saveObject(ctx, srcFilename, backupId, s.getBackupInstanceKey)
+func (s *SObjectBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error {
+	return s.saveObject(ctx, srcFile, fileSize, backupId, s.getBackupKey)
 }
 
-func (s *SObjectBackupStorage) saveObject(ctx context.Context, srcFilename string, id string, getKeyFunc func(string) string) error {
+func (s *SObjectBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error {
+	return s.saveObject(ctx, srcFile, fileSize, backupId, s.getBackupInstanceKey)
+}
+
+func (s *SObjectBackupStorage) saveObject(ctx context.Context, srcFile io.Reader, fileSize int64, id string, getKeyFunc func(string) string) error {
 	bucket, err := s.getBucket()
 	if err != nil {
 		return errors.Wrap(err, "getBucket")
 	}
-	fileInfo, err := os.Stat(srcFilename)
-	if err != nil {
-		return errors.Wrapf(err, "stat %s", srcFilename)
-	}
-	file, err := os.Open(srcFilename)
-	if err != nil {
-		return errors.Wrapf(err, "Open %s", srcFilename)
-	}
-	defer file.Close()
 
-	err = cloudprovider.UploadObject(ctx, bucket, getKeyFunc(id), 200*1024*1024, file, fileInfo.Size(), cloudprovider.ACLPrivate, "", nil, false)
+	err = cloudprovider.UploadObject(ctx, bucket, getKeyFunc(id), 200*1024*1024, srcFile, fileSize, cloudprovider.ACLPrivate, "", nil, false)
 	if err != nil {
-		return errors.Wrapf(err, "UploadObject %s %s", srcFilename, getKeyFunc(id))
+		return errors.Wrapf(err, "UploadObject %d %s", fileSize, getKeyFunc(id))
 	}
 
 	return nil
@@ -208,3 +237,21 @@ func (s *SObjectBackupStorage) IsOnline() (bool, string, error) {
 	}
 	return exist, "", nil
 }
+
+func (s *SObjectBackupStorage) GetExternalAccessUrl(backupId string) (string, error) {
+	var bucket cloudprovider.ICloudBucket
+	var err error
+	bucket, err = s.getExtBucket()
+	if err != nil {
+		log.Errorf("SObjectBackupStorage.GetExternalAccessUrl.getExtBucket: %v", err)
+		bucket, err = s.getBucket()
+		if err != nil {
+			return "", errors.Wrap(err, "getBucket")
+		}
+	}
+	url, err := bucket.GetTempUrl(http.MethodGet, s.getBackupKey(backupId), 6*time.Hour)
+	if err != nil {
+		return "", errors.Wrap(err, "GetTempUrl")
+	}
+	return url, nil
+}

pkg/hostman/storageman/storage_lvm.go

@@ -583,7 +583,7 @@ func (s *SLVMStorage) StorageBackup(ctx context.Context, params *SStorageBackup)
 		return nil, err
 	}
 	backupPath := params.BackupLocalPath
-	err = backupStorage.SaveBackupFrom(ctx, backupPath, params.BackupId)
+	err = backupstorage.SaveBackupFromFile(ctx, backupPath, params.BackupId, backupStorage)
 	if err != nil {
 		return nil, err
 	}

pkg/mcclient/options/compute/backup.go

@@ -15,7 +15,10 @@
 package compute
 
 import (
+	"os"
+
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -147,6 +150,8 @@ type BackupStorageCreateOptions struct {
 	ObjectSecret    string `help:"object storage secret, required when storage_type is object"`
 	ObjectSignVer   string `help:"object storage signing alogirithm version, optional" choices:"v2|v4"`
 
+	ObjectBucketUrlExt string `help:"object storage external access url, optional"`
+
 	CapacityMb int `help:"capacity, unit mb"`
 }
 
@@ -164,6 +169,8 @@ type BackupStorageUpdateOptions struct {
 	ObjectAccessKey string `help:"object storage access key, required when storage_type is object"`
 	ObjectSecret    string `help:"object storage secret, required when storage_type is object"`
 	ObjectSignVer   string `help:"object storage signing alogirithm version, optional" choices:"v2|v4"`
+
+	ObjectBucketUrlExt string `help:"object storage external access url, optional"`
 }
 
 func (opts *BackupStorageUpdateOptions) Params() (jsonutils.JSONObject, error) {
@@ -268,3 +275,39 @@ func (opts HostBackupStorageJoinOptions) GetSlaveId() string {
 func (opts HostBackupStorageJoinOptions) Params() (jsonutils.JSONObject, error) {
 	return jsonutils.NewDict(), nil
 }
+
+type DiskBackupExportOptions struct {
+	DiskBackupIdOptions
+
+	Output string `help:"output file path" json:"output"`
+}
+
+type DiskBackupImportOptions struct {
+	NAME            string `help:"backup name" json:"name"`
+	BACKUPSTORAGEID string `help:"backup storage id" json:"backup_storage_id"`
+	IMPORT          string `help:"backup info" json:"-"`
+
+	TenantId string `help:"tenant id" json:"tenant_id"`
+}
+
+func (opts *DiskBackupImportOptions) Params() (jsonutils.JSONObject, error) {
+	content, err := os.ReadFile(opts.IMPORT)
+	if err != nil {
+		return nil, errors.Wrap(err, "os.ReadFile")
+	}
+	inputJson, err := jsonutils.Parse(content)
+	if err != nil {
+		return nil, errors.Wrap(err, "jsonutils.Parse")
+	}
+	info := &computeapi.DiskBackupImportInput{}
+	err = inputJson.Unmarshal(info)
+	if err != nil {
+		return nil, errors.Wrap(err, "jsonutils.Unmarshal")
+	}
+	info.Name = opts.NAME
+	info.BackupStorageId = opts.BACKUPSTORAGEID
+	if len(opts.TenantId) > 0 {
+		info.ProjectId = opts.TenantId
+	}
+	return jsonutils.Marshal(info), nil
+}