From 5fc2174eebca31384bc1acaeb7a686348245202d Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Mon, 10 Aug 2026 11:25:07 +0800 Subject: [PATCH] feature: support create disk backup from file (#25175) Co-authored-by: Qiu Jian --- pkg/apis/compute/backup.go | 7 + pkg/apis/compute/disk.go | 11 +- pkg/apis/compute/instance_backup.go | 11 ++ pkg/compute/models/backup_storage.go | 15 +- pkg/compute/models/diskbackups.go | 156 ++++++++++++++---- pkg/compute/models/disks.go | 3 +- pkg/compute/models/instance_backup.go | 11 +- pkg/compute/regiondrivers/kvm.go | 53 +++++- pkg/hostman/storageman/backup.go | 24 ++- .../backupstorage/backup_storage.go | 29 ++-- .../storageman/backupstorage/nfs/factory.go | 8 +- .../storageman/backupstorage/nfs/nfs.go | 65 +++++--- .../backupstorage/object/factory.go | 8 +- .../storageman/backupstorage/object/object.go | 67 ++++---- .../storageman/diskhandlers/diskhandler.go | 4 +- pkg/hostman/storageman/storage_local.go | 7 +- pkg/hostman/storageman/storage_lvm.go | 2 +- .../storagehandler/storagehandler.go | 36 +++- pkg/hostman/storageman/storagehelper.go | 27 ++- pkg/mcclient/options/compute/backup.go | 5 + 20 files changed, 371 insertions(+), 178 deletions(-) diff --git a/pkg/apis/compute/backup.go b/pkg/apis/compute/backup.go index 19f2822ac7..7a9419e3a3 100644 --- a/pkg/apis/compute/backup.go +++ b/pkg/apis/compute/backup.go @@ -136,6 +136,10 @@ type DiskBackupCreateInput struct { apis.VirtualResourceCreateInput apis.EncryptedResourceCreateInput + // swagger:ignore + SizeMb int `json:"size_mb"` + // path to find backup file, in case of create backup from a backup file + BackupFilePath string `json:"backup_file_path"` // description: disk id DiskId string `json:"disk_id"` // swagger:ignore @@ -165,6 +169,9 @@ type DiskBackupPackMetadata struct { // 操作系统类型 OsType string `json:"os_type"` DiskConfig *SBackupDiskConfig `json:"disk_config"` + + // 备份文件路径 + BackupFilePath string `json:"backup_file_path"` } type DiskBackupExportInfo struct { diff --git a/pkg/apis/compute/disk.go b/pkg/apis/compute/disk.go index ce4e9c91ad..40702cb783 100644 --- a/pkg/apis/compute/disk.go +++ b/pkg/apis/compute/disk.go @@ -358,11 +358,12 @@ type DiskAllocateInput struct { } type DiskAllocateFromBackupInput struct { - BackupId string `json:"backup_id"` - BackupStorageId string `json:"backup_storage_id"` - BackupStorageAccessInfo *jsonutils.JSONDict `json:"backup_storage_access_info"` - DiskConfig *DiskConfig `json:"disk_config"` - BackupAsTar *DiskBackupAsTarInput `json:"backup_as_tar"` + BackupId string `json:"backup_id"` + BackupStorageId string `json:"backup_storage_id"` + BackupStorageAccessInfo *SBackupStorageAccessInfo `json:"backup_storage_access_info"` + DiskConfig *DiskConfig `json:"disk_config"` + BackupAsTar *DiskBackupAsTarInput `json:"backup_as_tar"` + BackupFilePath string `json:"backup_file_path"` } type DiskDeleteInput struct { diff --git a/pkg/apis/compute/instance_backup.go b/pkg/apis/compute/instance_backup.go index f8fbf2ab6c..889e8a2797 100644 --- a/pkg/apis/compute/instance_backup.go +++ b/pkg/apis/compute/instance_backup.go @@ -95,6 +95,8 @@ type SSimpleBackup struct { EncryptKeyId string `json:"encrypt_key_id"` // 创建时间 CreatedAt time.Time `json:"created_at"` + // 备份文件路径 + BackupFilePath string `json:"backup_file_path"` } type InstanceBackupRecoveryInput struct { @@ -112,3 +114,12 @@ type InstanceBackupManagerCreateFromPackageInput struct { BackupStorageId string `json:"backup_storage_id"` PackageName string `json:"package_name"` } + +type SStoragePackInstanceBackup struct { + PackageName string `json:"package_name"` + BackupStorageId string `json:"backup_storage_id"` + BackupStorageAccessInfo *SBackupStorageAccessInfo `json:"backup_storage_access_info"` + BackupIds []string `json:"backup_ids"` + Metadata *InstanceBackupPackMetadata `json:"metadata"` + DiskBackups []SSimpleBackup `json:"disk_backups"` +} diff --git a/pkg/compute/models/backup_storage.go b/pkg/compute/models/backup_storage.go index 5c767f9089..0a22680076 100644 --- a/pkg/compute/models/backup_storage.go +++ b/pkg/compute/models/backup_storage.go @@ -396,9 +396,22 @@ func (bs *SBackupStorage) GetIBackupStorage() (backupstorage.IBackupStorage, err return nil, errors.Wrap(err, "GetAccessInfo") } log.Infof("GetIBackupStorage %s %s", bs.Id, accessInfo.String()) - ibs, err := backupstorage.GetBackupStorage(bs.Id, jsonutils.Marshal(accessInfo).(*jsonutils.JSONDict)) + ibs, err := backupstorage.GetBackupStorage(bs.Id, accessInfo) if err != nil { return nil, errors.Wrap(err, "GetBackupStorage") } return ibs, nil } + +func (manager *SBackupStorageManager) fetchItems(filterFunc func(q *sqlchemy.SQuery) *sqlchemy.SQuery) ([]SBackupStorage, error) { + q := manager.Query() + if filterFunc != nil { + q = filterFunc(q) + } + ret := make([]SBackupStorage, 0) + err := db.FetchModelObjects(manager, q, &ret) + if err != nil { + return nil, errors.Wrap(err, "FetchModelObjects") + } + return ret, nil +} diff --git a/pkg/compute/models/diskbackups.go b/pkg/compute/models/diskbackups.go index 12c5a8d6b6..3c9feeda04 100644 --- a/pkg/compute/models/diskbackups.go +++ b/pkg/compute/models/diskbackups.go @@ -60,7 +60,7 @@ type SDiskBackup struct { db.SEncryptedResource - DiskId string `width:"36" charset:"ascii" nullable:"true" create:"required" list:"user" index:"true"` + DiskId string `width:"36" charset:"ascii" nullable:"true" create:"optional" list:"user" index:"true"` BackupStorageId string `width:"36" charset:"ascii" nullable:"true" create:"required" list:"user" index:"true"` StorageId string `width:"36" charset:"ascii" nullable:"true" list:"user"` @@ -71,6 +71,8 @@ type SDiskBackup struct { // 操作系统类型 OsType string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"optional"` DiskConfig *SBackupDiskConfig + + BackupFilePath string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional"` } var DiskBackupManager *SDiskBackupManager @@ -229,6 +231,76 @@ func (db *SDiskBackup) GetRegionDriver() (IRegionDriver, error) { return cloudRegion.GetDriver(), nil } +func (dm *SDiskBackupManager) validateCreateDataFromBackupFile( + ctx context.Context, + userCred mcclient.TokenCredential, + ownerId mcclient.IIdentityProvider, + query jsonutils.JSONObject, + input api.DiskBackupCreateInput, +) (api.DiskBackupCreateInput, error) { + var backupStorages []*SBackupStorage + if len(input.DiskId) > 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "disk_id is not allowed") + } + if len(input.BackupStorageId) > 0 { + bsObj, err := BackupStorageManager.FetchByIdOrName(ctx, userCred, input.BackupStorageId) + if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + return input, httperrors.NewResourceNotFoundError2(BackupStorageManager.Keyword(), input.BackupStorageId) + } + if errors.Cause(err) == sqlchemy.ErrDuplicateEntry { + return input, httperrors.NewDuplicateResourceError(BackupStorageManager.Keyword(), input.BackupStorageId) + } + return input, httperrors.NewGeneralError(err) + } + backupStorages = append(backupStorages, bsObj.(*SBackupStorage)) + } else { + bsItems, err := BackupStorageManager.fetchItems(func(q *sqlchemy.SQuery) *sqlchemy.SQuery { + q = q.Equals("status", api.BACKUPSTORAGE_STATUS_ONLINE) + q = q.Equals("enabled", 1) + return q + }) + if err != nil { + return input, httperrors.NewGeneralError(err) + } + for i := range bsItems { + backupStorages = append(backupStorages, &bsItems[i]) + } + } + if len(backupStorages) == 0 { + return input, errors.Wrap(httperrors.ErrInputParameter, "no backup storage found") + } + var errs []error + input.SizeMb = -1 + for i := range backupStorages { + ibs, err := backupStorages[i].GetIBackupStorage() + if err != nil { + return input, errors.Wrap(err, "GetIBackupStorage") + } + exists, sizeBytes, offlineReason, err := ibs.IsBackupExists("", input.BackupFilePath) + if !exists { + if err != nil { + errs = append(errs, errors.Wrapf(err, "Backup storage error %s(%s)", backupStorages[i].GetName(), backupStorages[i].GetId())) + } + if len(offlineReason) > 0 { + errs = append(errs, errors.Wrapf(errors.ErrInvalidStatus, "Backup storage %s(%s) is offline: %s", backupStorages[i].GetName(), backupStorages[i].GetId(), offlineReason)) + } + errs = append(errs, errors.Wrapf(errors.ErrNotFound, "Backup file %s not found in backup storage %s(%s)", input.BackupFilePath, backupStorages[i].GetName(), backupStorages[i].GetId())) + } else { + input.BackupStorageId = backupStorages[i].GetId() + input.SizeMb = int(sizeBytes / 1024 / 1024) + break + } + } + if input.SizeMb == -1 { + if len(errs) > 0 { + return input, errors.NewAggregate(errs) + } + return input, errors.Wrap(httperrors.ErrInputParameter, "no backup storage found") + } + return input, nil +} + func (dm *SDiskBackupManager) ValidateCreateData( ctx context.Context, userCred mcclient.TokenCredential, @@ -236,6 +308,10 @@ func (dm *SDiskBackupManager) ValidateCreateData( query jsonutils.JSONObject, input api.DiskBackupCreateInput, ) (api.DiskBackupCreateInput, error) { + if len(input.BackupFilePath) > 0 { + // create disk backup from a backup file + return dm.validateCreateDataFromBackupFile(ctx, userCred, ownerId, query, input) + } if input.NeedEncrypt() { return input, errors.Wrap(httperrors.ErrInputParameter, "encryption should not be specified") } @@ -272,9 +348,7 @@ func (dm *SDiskBackupManager) ValidateCreateData( } return input, httperrors.NewGeneralError(err) } - if err != nil { - return input, err - } + bs := ibs.(*SBackupStorage) if bs.Status != api.BACKUPSTORAGE_STATUS_ONLINE { return input, httperrors.NewForbiddenError("can't backup guest to backup storage with status %s", bs.Status) @@ -329,36 +403,42 @@ func (db *SDiskBackup) CustomizeCreate(ctx context.Context, userCred mcclient.To if err != nil { return err } - diskObj, err := DiskManager.FetchById(db.DiskId) - if err != nil { - return errors.Wrap(err, "DiskManager.FetchById") + if len(db.DiskId) > 0 { + diskObj, err := DiskManager.FetchById(db.DiskId) + if err != nil { + return errors.Wrap(err, "DiskManager.FetchById") + } + disk := diskObj.(*SDisk) + db.DiskConfig = &SBackupDiskConfig{ + DiskConfig: *disk.ToDiskConfig(), + Name: disk.GetName(), + BackupAsTar: input.BackupAsTar, + } + db.DiskType = disk.DiskType + db.DiskSizeMb = disk.DiskSize + db.OsArch = disk.OsArch + db.StorageId = disk.StorageId + db.DomainId = disk.DomainId + db.ProjectId = disk.ProjectId } - disk := diskObj.(*SDisk) - db.DiskConfig = &SBackupDiskConfig{ - DiskConfig: *disk.ToDiskConfig(), - Name: disk.GetName(), - BackupAsTar: input.BackupAsTar, - } - db.DiskType = disk.DiskType - db.DiskSizeMb = disk.DiskSize - db.OsArch = disk.OsArch - db.StorageId = disk.StorageId - db.DomainId = disk.DomainId - db.ProjectId = disk.ProjectId return nil } func (db *SDiskBackup) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) { db.SVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data) - disk, err := db.GetDisk() - if err != nil { - log.Errorf("unable to GetDisk: %s", err.Error()) + if len(db.DiskId) > 0 { + disk, err := db.GetDisk() + if err != nil { + log.Errorf("unable to GetDisk: %s", err.Error()) + } + err = disk.InheritTo(ctx, userCred, db) + if err != nil { + log.Errorf("unable to inherit from disk %s to backup %s: %s", disk.GetId(), db.GetId(), err.Error()) + } + db.StartBackupCreateTask(ctx, userCred, nil, "") + } else { + db.SetStatus(ctx, userCred, api.BACKUP_STATUS_READY, "import from backup file") } - err = disk.InheritTo(ctx, userCred, db) - if err != nil { - log.Errorf("unable to inherit from disk %s to backup %s: %s", disk.GetId(), db.GetId(), err.Error()) - } - db.StartBackupCreateTask(ctx, userCred, nil, "") } func (db *SDiskBackup) StartBackupCreateTask(ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, parentTaskId string) error { @@ -568,6 +648,8 @@ func (diskBackup *SDiskBackup) PackMetadata() *api.DiskBackupPackMetadata { DiskConfig: diskBackup.DiskConfig.DiskConfig, Name: diskBackup.DiskConfig.Name, }, + // 备份文件路径 + BackupFilePath: diskBackup.BackupFilePath, } } @@ -591,6 +673,7 @@ func (manager *SDiskBackupManager) CreateFromPackMetadata(ctx context.Context, o backup.Name = name backup.Id = id backup.Status = api.BACKUP_STATUS_READY + backup.BackupFilePath = metadata.BackupFilePath err := DiskBackupManager.TableSpec().Insert(ctx, backup) if err != nil { return nil, err @@ -631,7 +714,7 @@ func (diskBackup *SDiskBackup) GetDetailsExportInfo(ctx context.Context, userCre return nil, errors.Wrap(err, "GetIBackupStorage") } - exportInfo.AccessUrl, err = ibs.GetExternalAccessUrl(diskBackup.Id) + exportInfo.AccessUrl, err = ibs.GetExternalAccessUrl(diskBackup.Id, diskBackup.BackupFilePath) if err != nil { log.Errorf("SDiskBackup %s(%s) GetExternalAccessUrl fail: %v", diskBackup.GetName(), diskBackup.GetId(), err) } @@ -744,10 +827,25 @@ func (diskBackup *SDiskBackup) DoImport(ctx context.Context, userCred mcclient.T } defer resp.Body.Close() - err = ibs.SaveBackupFrom(ctx, resp.Body, resp.ContentLength, diskBackup.Id) + err = ibs.SaveBackupFrom(ctx, resp.Body, resp.ContentLength, diskBackup.Id, "") if err != nil { return errors.Wrap(err, "SaveBackupFrom") } return nil } + +func (diskBackup SDiskBackup) ToSimpleBackup() api.SSimpleBackup { + return api.SSimpleBackup{ + Id: diskBackup.Id, + Name: diskBackup.Name, + SizeMb: diskBackup.SizeMb, + DiskSizeMb: diskBackup.DiskSizeMb, + DiskType: diskBackup.DiskType, + Status: diskBackup.Status, + EncryptKeyId: diskBackup.EncryptKeyId, + CreatedAt: diskBackup.CreatedAt, + + BackupFilePath: diskBackup.BackupFilePath, + } +} diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 0bca7b5589..5d804dd806 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -920,9 +920,10 @@ func (self *SDisk) getDiskAllocateFromBackupInput(ctx context.Context, backupId return &api.DiskAllocateFromBackupInput{ BackupId: backupId, BackupStorageId: bs.GetId(), - BackupStorageAccessInfo: jsonutils.Marshal(accessInfo).(*jsonutils.JSONDict), + BackupStorageAccessInfo: accessInfo, DiskConfig: &backup.DiskConfig.DiskConfig, BackupAsTar: backup.DiskConfig.BackupAsTar, + BackupFilePath: backup.BackupFilePath, }, nil } diff --git a/pkg/compute/models/instance_backup.go b/pkg/compute/models/instance_backup.go index 7dca373479..0c30f5d612 100644 --- a/pkg/compute/models/instance_backup.go +++ b/pkg/compute/models/instance_backup.go @@ -225,16 +225,7 @@ func (self *SInstanceBackup) getMoreDetails(userCred mcclient.TokenCredential, o backups, _ := self.GetBackups() out.DiskBackups = []api.SSimpleBackup{} for i := 0; i < len(backups); i++ { - out.DiskBackups = append(out.DiskBackups, api.SSimpleBackup{ - Id: backups[i].Id, - Name: backups[i].Name, - SizeMb: backups[i].SizeMb, - DiskSizeMb: backups[i].DiskSizeMb, - DiskType: backups[i].DiskType, - Status: backups[i].Status, - EncryptKeyId: backups[i].EncryptKeyId, - CreatedAt: backups[i].CreatedAt, - }) + out.DiskBackups = append(out.DiskBackups, backups[i].ToSimpleBackup()) } out.Size = self.SizeMb * 1024 * 1024 return out diff --git a/pkg/compute/regiondrivers/kvm.go b/pkg/compute/regiondrivers/kvm.go index 5de646671f..223feba7be 100644 --- a/pkg/compute/regiondrivers/kvm.go +++ b/pkg/compute/regiondrivers/kvm.go @@ -987,23 +987,29 @@ func (self *SKVMRegionDriver) RequestPackInstanceBackup(ctx context.Context, ib for i := range backupIds { backupIds[i] = backups[i].GetId() } + diskBackups := make([]api.SSimpleBackup, len(backups)) + for i := range diskBackups { + diskBackups[i] = backups[i].ToSimpleBackup() + } metadata, err := ib.PackMetadata(ctx, task.GetUserCred()) if err != nil { return errors.Wrap(err, "unable to PackMetadata") } url := fmt.Sprintf("%s/storages/pack-instance-backup", host.ManagerUri) - body := jsonutils.NewDict() - body.Set("package_name", jsonutils.NewString(packageName)) - body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId())) accessInfo, err := backupStorage.GetAccessInfo() if err != nil { return errors.Wrap(err, "GetAccessInfo") } - body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) - body.Set("backup_ids", jsonutils.Marshal(backupIds)) - body.Set("metadata", jsonutils.Marshal(metadata)) + body := api.SStoragePackInstanceBackup{ + PackageName: packageName, + BackupStorageId: backupStorage.GetId(), + BackupStorageAccessInfo: accessInfo, + DiskBackups: diskBackups, + BackupIds: backupIds, + Metadata: metadata, + } header := task.GetTaskRequestHeader() - _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) + _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, jsonutils.Marshal(body), false) if err != nil { return errors.Wrap(err, "unable to pack instancebackup") } @@ -1045,6 +1051,24 @@ func (self *SKVMRegionDriver) RequestSyncBackupStorageStatus(ctx context.Context taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { host, err := models.HostManager.GetEnabledKvmHostForBackupStorage(bs) if err != nil { + if errors.Cause(err) == sql.ErrNoRows { + // try to detect the backup storage status from region + ibs, err := bs.GetIBackupStorage() + if err != nil { + return nil, errors.Wrap(err, "GetIBackupStorage") + } + online, reason, err := ibs.IsOnline() + if err != nil { + return nil, errors.Wrap(err, "IsOnline") + } + var statusStr string + if !online { + statusStr = api.BACKUPSTORAGE_STATUS_OFFLINE + } else { + statusStr = api.BACKUPSTORAGE_STATUS_ONLINE + } + return nil, bs.SetStatus(ctx, userCred, statusStr, reason) + } return nil, errors.Wrap(err, "GetEnabledKvmHostForBackupStorage") } url := fmt.Sprintf("%s/storages/sync-backup-storage", host.ManagerUri) @@ -1135,6 +1159,9 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u body := jsonutils.NewDict() body.Set("backup_id", jsonutils.NewString(backup.GetId())) body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId())) + if len(backup.BackupFilePath) > 0 { + body.Set("backup_file_path", jsonutils.NewString(backup.BackupFilePath)) + } accessInfo, err := backupStorage.GetAccessInfo() if err != nil { return nil, errors.Wrap(err, "GetAccessInfo") @@ -1152,7 +1179,11 @@ func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, u } else { backupStatus = api.BACKUP_STATUS_UNKNOWN } - return nil, backup.SetStatus(ctx, userCred, backupStatus, "sync status") + reason, _ := res.GetString("reason") + if len(reason) == 0 { + reason = "sync status" + } + return nil, backup.SetStatus(ctx, userCred, backupStatus, reason) }) return nil } @@ -1350,6 +1381,9 @@ func (self *SKVMRegionDriver) RequestDeleteBackup(ctx context.Context, backup *m return errors.Wrap(err, "GetAccessInfo") } body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) + if len(backup.BackupFilePath) > 0 { + body.Set("backup_file_path", jsonutils.NewString(backup.BackupFilePath)) + } header := task.GetTaskRequestHeader() _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false) if err != nil { @@ -1394,6 +1428,9 @@ func (self *SKVMRegionDriver) RequestCreateBackup(ctx context.Context, backup *m return errors.Wrap(err, "GetAccessInfo") } body.Set("backup_storage_access_info", jsonutils.Marshal(accessInfo)) + if len(backup.BackupFilePath) > 0 { + body.Set("backup_file_path", jsonutils.NewString(backup.BackupFilePath)) + } if len(backup.EncryptKeyId) > 0 { body.Set("encrypt_key_id", jsonutils.NewString(backup.EncryptKeyId)) } diff --git a/pkg/hostman/storageman/backup.go b/pkg/hostman/storageman/backup.go index e2ac0935ca..ec1e782df2 100644 --- a/pkg/hostman/storageman/backup.go +++ b/pkg/hostman/storageman/backup.go @@ -117,7 +117,7 @@ func doBackupDisk(ctx context.Context, snapshotPath string, diskBackup *SDiskBac return 0, errors.Wrap(err, "GetBackupStorage") } - err = backupstorage.SaveBackupFromFile(ctx, backupPath, diskBackup.BackupId, backupStorage) + err = backupstorage.SaveBackupFromFile(ctx, backupPath, diskBackup.BackupId, diskBackup.BackupFilePath, backupStorage) if err != nil { return 0, errors.Wrap(err, "SaveBackupFrom") } @@ -143,7 +143,7 @@ func doRestoreDisk(ctx context.Context, dc IDiskCreator, input *SDiskCreateByDis return errors.Wrap(err, "GetBackupStorage") } backupPath := path.Join(backupTmpDir, diskInfo.Backup.BackupId) - err = backupStorage.RestoreBackupTo(ctx, backupPath, diskInfo.Backup.BackupId) + err = backupStorage.RestoreBackupTo(ctx, backupPath, diskInfo.Backup.BackupId, diskInfo.Backup.BackupFilePath) if err != nil { return errors.Wrapf(err, "Restore backup %s to %s", diskInfo.Backup.BackupId, backupPath) } @@ -293,7 +293,7 @@ const ( PackageMetadataFilename = "metadata" ) -func DoInstancePackBackup(ctx context.Context, backupInfo SStoragePackInstanceBackup) (string, error) { +func DoInstancePackBackup(ctx context.Context, backupInfo api.SStoragePackInstanceBackup) (string, error) { backupTmpDir, err := EnsureBackupDir() if err != nil { return "", errors.Wrap(err, "EnsureBackupDir") @@ -314,11 +314,21 @@ func DoInstancePackBackup(ctx context.Context, backupInfo SStoragePackInstanceBa return "", errors.Wrapf(err, "mkdir %s failed: %s", packagePath, output) } } - { + if len(backupInfo.DiskBackups) > 0 { + // download disk files + for i := range backupInfo.DiskBackups { + backup := backupInfo.DiskBackups[i] + packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i)) + err := backupStorage.RestoreBackupTo(ctx, packageDiskPath, backup.Id, backup.BackupFilePath) + if err != nil { + return "", errors.Wrapf(err, "RestoreBackupTo %s %s", backup.Id, packageDiskPath) + } + } + } else if len(backupInfo.BackupIds) > 0 { // for backward compatibility // download disk files for i, backupId := range backupInfo.BackupIds { packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i)) - err := backupStorage.RestoreBackupTo(ctx, packageDiskPath, backupId) + err := backupStorage.RestoreBackupTo(ctx, packageDiskPath, backupId, "") if err != nil { return "", errors.Wrapf(err, "RestoreBackupTo %s %s", backupId, packageDiskPath) } @@ -350,7 +360,7 @@ func DoInstancePackBackup(ctx context.Context, backupInfo SStoragePackInstanceBa } else { finalPackageFileName = fmt.Sprintf("%s-%d.tar", backupInfo.PackageName, tried) } - exists, _, err := backupStorage.IsBackupInstanceExists(finalPackageFileName) + exists, _, _, err := backupStorage.IsBackupInstanceExists(finalPackageFileName) if err != nil { return "", errors.Wrap(err, "IsBackupInstanceExists") } @@ -432,7 +442,7 @@ func DoInstanceUnpackBackup(ctx context.Context, backupInfo SStorageUnpackInstan backupId := db.DefaultUUIDGenerator() backupIds[i] = backupId packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i)) - err := backupstorage.SaveBackupFromFile(ctx, packageDiskPath, backupId, backupStorage) + err := backupstorage.SaveBackupFromFile(ctx, packageDiskPath, backupId, "", backupStorage) if err != nil { return nil, nil, errors.Wrapf(err, "SaveBackupFrom %s %s", packageDiskPath, backupId) } diff --git a/pkg/hostman/storageman/backupstorage/backup_storage.go b/pkg/hostman/storageman/backupstorage/backup_storage.go index 6a25360d34..2755799703 100644 --- a/pkg/hostman/storageman/backupstorage/backup_storage.go +++ b/pkg/hostman/storageman/backupstorage/backup_storage.go @@ -20,23 +20,24 @@ import ( "os" "sync" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis/compute" ) type IBackupStorageFactory interface { - NewBackupStore(storeId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) + NewBackupStore(storeId string, backupStorageAccessInfo *compute.SBackupStorageAccessInfo) (IBackupStorage, error) } type IBackupStorage interface { // 从指定路径拷贝磁盘文件到备份存储 - SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupId string) error + SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupId string, backupFilePath string) error // 将备份backupId对应的备份文件拷贝到指定的文件路径 - RestoreBackupTo(ctx context.Context, targetFilename string, backupId string) error + RestoreBackupTo(ctx context.Context, targetFilename string, backupId string, backupFilePath string) error // 删除备份 - RemoveBackup(ctx context.Context, backupId string) error + RemoveBackup(ctx context.Context, backupId string, backupFilePath string) error // 备份是否存在 - IsBackupExists(backupId string) (bool, string, error) + IsBackupExists(backupId string, backupFilePath string) (bool, int64, string, error) // 从指定路径拷贝主机备份文件到备份存储 SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupInstanceId string) error @@ -45,7 +46,7 @@ type IBackupStorage interface { // 删除备份 RemoveBackupInstance(ctx context.Context, backupInstanceId string) error // 备份是否存在 - IsBackupInstanceExists(backupInstanceId string) (bool, string, error) + IsBackupInstanceExists(bakcupInstanceFilePath string) (bool, int64, string, error) // ConvertTo(destPath string, format qemuimgfmt.TImageFormat, backupId string) error // ConvertFrom(srcPath string, format qemuimgfmt.TImageFormat, backupId string) (int, error) @@ -56,7 +57,7 @@ type IBackupStorage interface { IsOnline() (bool, string, error) // 获取外部访问地址 - GetExternalAccessUrl(backupId string) (string, error) + GetExternalAccessUrl(backupId string, backupFilePath string) (string, error) } var factories []IBackupStorageFactory @@ -72,7 +73,7 @@ func RegisterFactory(factory IBackupStorageFactory) { factories = append(factories, factory) } -func newBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) { +func newBackupStorage(backupStroageId string, backupStorageAccessInfo *compute.SBackupStorageAccessInfo) (IBackupStorage, error) { errs := make([]error, 0) for _, factory := range factories { store, err := factory.NewBackupStore(backupStroageId, backupStorageAccessInfo) @@ -85,7 +86,7 @@ func newBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils return nil, errors.NewAggregate(errs) } -func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) { +func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *compute.SBackupStorageAccessInfo) (IBackupStorage, error) { backupStorageLock.Lock() defer backupStorageLock.Unlock() @@ -101,7 +102,7 @@ func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils } } -func SaveBackupFromFile(ctx context.Context, srcFilename string, bakcupId string, storage IBackupStorage) error { +func SaveBackupFromFile(ctx context.Context, srcFilename string, bakcupId string, backupFilePath string, storage IBackupStorage) error { fileInfo, err := os.Stat(srcFilename) if err != nil { return errors.Wrapf(err, "stat %s", srcFilename) @@ -112,10 +113,10 @@ func SaveBackupFromFile(ctx context.Context, srcFilename string, bakcupId string } defer file.Close() - return storage.SaveBackupFrom(ctx, file, fileInfo.Size(), bakcupId) + return storage.SaveBackupFrom(ctx, file, fileInfo.Size(), bakcupId, backupFilePath) } -func SaveBackupInstanceFromFile(ctx context.Context, srcFilename string, bakcupInstanceId string, storage IBackupStorage) error { +func SaveBackupInstanceFromFile(ctx context.Context, srcFilename string, bakcupInstanceFilePath string, storage IBackupStorage) error { fileInfo, err := os.Stat(srcFilename) if err != nil { return errors.Wrapf(err, "stat %s", srcFilename) @@ -125,5 +126,5 @@ func SaveBackupInstanceFromFile(ctx context.Context, srcFilename string, bakcupI return errors.Wrapf(err, "Open %s", srcFilename) } defer file.Close() - return storage.SaveBackupInstanceFrom(ctx, file, fileInfo.Size(), bakcupInstanceId) + return storage.SaveBackupInstanceFrom(ctx, file, fileInfo.Size(), bakcupInstanceFilePath) } diff --git a/pkg/hostman/storageman/backupstorage/nfs/factory.go b/pkg/hostman/storageman/backupstorage/nfs/factory.go index 9484f0296b..a4a52098ab 100644 --- a/pkg/hostman/storageman/backupstorage/nfs/factory.go +++ b/pkg/hostman/storageman/backupstorage/nfs/factory.go @@ -15,7 +15,6 @@ package nfs import ( - "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -25,12 +24,7 @@ import ( type sNfsBackupStorageFactory struct{} -func (factory *sNfsBackupStorageFactory) NewBackupStore(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (backupstorage.IBackupStorage, error) { - accessInfo := api.SBackupStorageAccessInfo{} - err := backupStorageAccessInfo.Unmarshal(&accessInfo) - if err != nil { - return nil, errors.Wrap(err, "Unmarshal access info") - } +func (factory *sNfsBackupStorageFactory) NewBackupStore(backupStroageId string, accessInfo *api.SBackupStorageAccessInfo) (backupstorage.IBackupStorage, error) { if len(accessInfo.NfsHost) == 0 { return nil, errors.Wrap(httperrors.ErrInputParameter, "need nfs_host in backup_storage_access_info") } diff --git a/pkg/hostman/storageman/backupstorage/nfs/nfs.go b/pkg/hostman/storageman/backupstorage/nfs/nfs.go index 62c6ec3e6d..ff911e9ef4 100644 --- a/pkg/hostman/storageman/backupstorage/nfs/nfs.go +++ b/pkg/hostman/storageman/backupstorage/nfs/nfs.go @@ -57,7 +57,10 @@ func (s *SNFSBackupStorage) getBackupDir() string { return path.Join(s.Path, "backups") } -func (s *SNFSBackupStorage) getBackupDiskPath(backupId string) string { +func (s *SNFSBackupStorage) getBackupDiskPath(backupId string, backupFilePath string) string { + if len(backupFilePath) > 0 { + return backupFilePath + } return path.Join(s.getBackupDir(), backupId) } @@ -65,7 +68,7 @@ func (s *SNFSBackupStorage) getPackageDir() string { return path.Join(s.Path, "backuppacks") } -func (s *SNFSBackupStorage) getBackupInstancePath(backupInstanceId string) string { +func (s *SNFSBackupStorage) getBackupInstancePath(backupInstanceId string, backupFilePath string) string { return path.Join(s.getPackageDir(), backupInstanceId) } @@ -124,22 +127,22 @@ func (s *SNFSBackupStorage) unMount() error { return nil } -func (s *SNFSBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveFile(ctx, srcFile, fileSize, backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string, backupFilePath string) error { + return s.saveFile(ctx, srcFile, fileSize, backupId, backupFilePath, s.getBackupDiskPath) } func (s *SNFSBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveFile(ctx, srcFile, fileSize, backupId, s.getBackupDiskPath) + return s.saveFile(ctx, srcFile, fileSize, backupId, "", s.getBackupInstancePath) } -func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFile io.Reader, fileSize int64, id string, getPathFunc func(string) string) error { +func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFile io.Reader, fileSize int64, id string, backupFilePath string, getPathFunc func(string, string) string) error { err := s.checkAndMount() if err != nil { return errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - targetFilename := getPathFunc(id) + targetFilename := getPathFunc(id, backupFilePath) targetFile, err := os.Create(targetFilename) if err != nil { @@ -154,22 +157,22 @@ func (s *SNFSBackupStorage) saveFile(ctx context.Context, srcFile io.Reader, fil return nil } -func (s *SNFSBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreFile(ctx, targetFilename, backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string, backupFilePath string) error { + return s.restoreFile(ctx, targetFilename, backupId, backupFilePath, s.getBackupDiskPath) } func (s *SNFSBackupStorage) RestoreBackupInstanceTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreFile(ctx, targetFilename, backupId, s.getBackupInstancePath) + return s.restoreFile(ctx, targetFilename, backupId, "", s.getBackupInstancePath) } -func (s *SNFSBackupStorage) restoreFile(ctx context.Context, targetFilename string, id string, getPathFunc func(string) string) error { +func (s *SNFSBackupStorage) restoreFile(ctx context.Context, targetFilename string, id string, backupFilePath string, getPathFunc func(string, string) string) error { err := s.checkAndMount() if err != nil { return errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - srcFilename := getPathFunc(id) + srcFilename := getPathFunc(id, backupFilePath) if output, err := procutils.NewCommand("cp", srcFilename, targetFilename).Output(); err != nil { log.Errorf("unable to cp %s to %s: %s", srcFilename, targetFilename, output) return errors.Wrapf(err, "cp %s to %s failed and output is %q", srcFilename, targetFilename, output) @@ -177,22 +180,22 @@ func (s *SNFSBackupStorage) restoreFile(ctx context.Context, targetFilename stri return nil } -func (s *SNFSBackupStorage) RemoveBackup(ctx context.Context, backupId string) error { - return s.removeFile(ctx, backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) RemoveBackup(ctx context.Context, backupId string, backupFilePath string) error { + return s.removeFile(ctx, backupId, backupFilePath, s.getBackupDiskPath) } func (s *SNFSBackupStorage) RemoveBackupInstance(ctx context.Context, backupId string) error { - return s.removeFile(ctx, backupId, s.getBackupInstancePath) + return s.removeFile(ctx, backupId, "", s.getBackupInstancePath) } -func (s *SNFSBackupStorage) removeFile(ctx context.Context, id string, getPathFunc func(id string) string) error { +func (s *SNFSBackupStorage) removeFile(ctx context.Context, id string, backupFilePath string, getPathFunc func(string, string) string) error { err := s.checkAndMount() if err != nil { return errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - filename := getPathFunc(id) + filename := getPathFunc(id, backupFilePath) if !fileutils2.Exists(filename) { return nil } @@ -203,26 +206,34 @@ func (s *SNFSBackupStorage) removeFile(ctx context.Context, id string, getPathFu return nil } -func (s *SNFSBackupStorage) IsBackupExists(backupId string) (bool, string, error) { - return s.isFileExists(backupId, s.getBackupDiskPath) +func (s *SNFSBackupStorage) IsBackupExists(backupId string, backupFilePath string) (bool, int64, string, error) { + return s.isFileExists(backupId, s.getBackupDiskPath, backupFilePath) } -func (s *SNFSBackupStorage) IsBackupInstanceExists(backupId string) (bool, string, error) { - return s.isFileExists(backupId, s.getBackupInstancePath) +func (s *SNFSBackupStorage) IsBackupInstanceExists(backupId string) (bool, int64, string, error) { + return s.isFileExists(backupId, s.getBackupInstancePath, "") } -func (s *SNFSBackupStorage) isFileExists(id string, getPathFunc func(id string) string) (bool, string, error) { +func (s *SNFSBackupStorage) isFileExists(id string, getPathFunc func(string, string) string, backupFilePath string) (bool, int64, string, error) { err := s.checkAndMount() if err != nil { if errors.Cause(err) == ErrorBackupStorageOffline { - return false, err.Error(), nil + return false, -1, err.Error(), nil } - return false, "", errors.Wrap(err, "unable to checkAndMount") + return false, -1, "", errors.Wrap(err, "unable to checkAndMount") } defer s.unMount() - filename := getPathFunc(id) - return fileutils2.Exists(filename), "", nil + filename := getPathFunc(id, backupFilePath) + exists := fileutils2.Exists(filename) + if !exists { + return false, -1, "", nil + } + stat, err := os.Stat(filename) + if err != nil { + return false, -1, "", errors.Wrap(err, "os.Stat") + } + return true, stat.Size(), "", nil } func (s *SNFSBackupStorage) IsOnline() (bool, string, error) { @@ -237,6 +248,6 @@ func (s *SNFSBackupStorage) IsOnline() (bool, string, error) { return true, "", nil } -func (s *SNFSBackupStorage) GetExternalAccessUrl(backupId string) (string, error) { +func (s *SNFSBackupStorage) GetExternalAccessUrl(backupId string, backupFilePath string) (string, error) { return "", errors.ErrNotSupported } diff --git a/pkg/hostman/storageman/backupstorage/object/factory.go b/pkg/hostman/storageman/backupstorage/object/factory.go index 45c5ef6c42..2a557b31b5 100644 --- a/pkg/hostman/storageman/backupstorage/object/factory.go +++ b/pkg/hostman/storageman/backupstorage/object/factory.go @@ -16,7 +16,6 @@ package object import ( "yunion.io/x/cloudmux/pkg/multicloud/objectstore" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -26,12 +25,7 @@ import ( type sObjectBackupStorageFactory struct{} -func (factory *sObjectBackupStorageFactory) NewBackupStore(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (backupstorage.IBackupStorage, error) { - accessInfo := api.SBackupStorageAccessInfo{} - err := backupStorageAccessInfo.Unmarshal(&accessInfo) - if err != nil { - return nil, errors.Wrap(err, "Unmarshal access info") - } +func (factory *sObjectBackupStorageFactory) NewBackupStore(backupStroageId string, accessInfo *api.SBackupStorageAccessInfo) (backupstorage.IBackupStorage, error) { if len(accessInfo.ObjectBucketUrl) == 0 { return nil, errors.Wrap(httperrors.ErrInputParameter, "need object_bucket_url in backup_storage_access_info") } diff --git a/pkg/hostman/storageman/backupstorage/object/object.go b/pkg/hostman/storageman/backupstorage/object/object.go index 05bbcda801..64b63cb7d2 100644 --- a/pkg/hostman/storageman/backupstorage/object/object.go +++ b/pkg/hostman/storageman/backupstorage/object/object.go @@ -109,18 +109,21 @@ func parseBucketUrl(bucketUrl string) (string, string, error) { const backupPathPrefix = "backups" const backupInstancePathPrefix = "backuppacks" -func (s *SObjectBackupStorage) getBackupKey(backupId string) string { +func (s *SObjectBackupStorage) getBackupKey(backupId string, backupFilePath string) string { + if len(backupFilePath) > 0 { + return backupFilePath + } return fmt.Sprintf("%s/%s", backupPathPrefix, backupId) } -func (s *SObjectBackupStorage) getBackupInstanceKey(backupInstancePackName string) string { +func (s *SObjectBackupStorage) getBackupInstanceKey(backupInstancePackName string, _ string) string { return fmt.Sprintf("%s/%s", backupInstancePathPrefix, backupInstancePackName) } func (s *SObjectBackupStorage) getBucket() (cloudprovider.ICloudBucket, error) { bucket, err := s.store.GetIRegion().GetIBucketByName(s.bucket) if err != nil { - return nil, errors.Wrap(err, "IBucketExist") + return nil, errors.Wrapf(err, "IBucketExist %s", s.bucket) } return bucket, nil } @@ -136,42 +139,42 @@ func (s *SObjectBackupStorage) getExtBucket() (cloudprovider.ICloudBucket, error return bucket, nil } -func (s *SObjectBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveObject(ctx, srcFile, fileSize, backupId, s.getBackupKey) +func (s *SObjectBackupStorage) SaveBackupFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string, backupFilePath string) error { + return s.saveObject(ctx, srcFile, fileSize, backupId, backupFilePath, s.getBackupKey) } -func (s *SObjectBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, backupId string) error { - return s.saveObject(ctx, srcFile, fileSize, backupId, s.getBackupInstanceKey) +func (s *SObjectBackupStorage) SaveBackupInstanceFrom(ctx context.Context, srcFile io.Reader, fileSize int64, bakcupInstanceFilePath string) error { + return s.saveObject(ctx, srcFile, fileSize, bakcupInstanceFilePath, "", s.getBackupInstanceKey) } -func (s *SObjectBackupStorage) saveObject(ctx context.Context, srcFile io.Reader, fileSize int64, id string, getKeyFunc func(string) string) error { +func (s *SObjectBackupStorage) saveObject(ctx context.Context, srcFile io.Reader, fileSize int64, id string, backupFilePath string, getKeyFunc func(string, string) string) error { bucket, err := s.getBucket() if err != nil { return errors.Wrap(err, "getBucket") } - err = cloudprovider.UploadObject(ctx, bucket, getKeyFunc(id), 200*1024*1024, srcFile, fileSize, cloudprovider.ACLPrivate, "", nil, false) + err = cloudprovider.UploadObject(ctx, bucket, getKeyFunc(id, backupFilePath), 200*1024*1024, srcFile, fileSize, cloudprovider.ACLPrivate, "", nil, false) if err != nil { - return errors.Wrapf(err, "UploadObject %d %s", fileSize, getKeyFunc(id)) + return errors.Wrapf(err, "UploadObject %d %s", fileSize, getKeyFunc(id, backupFilePath)) } return nil } -func (s *SObjectBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreObject(ctx, targetFilename, backupId, s.getBackupKey) +func (s *SObjectBackupStorage) RestoreBackupTo(ctx context.Context, targetFilename string, backupId string, backupFilePath string) error { + return s.restoreObject(ctx, targetFilename, backupId, backupFilePath, s.getBackupKey) } func (s *SObjectBackupStorage) RestoreBackupInstanceTo(ctx context.Context, targetFilename string, backupId string) error { - return s.restoreObject(ctx, targetFilename, backupId, s.getBackupInstanceKey) + return s.restoreObject(ctx, targetFilename, backupId, "", s.getBackupInstanceKey) } -func (s *SObjectBackupStorage) restoreObject(ctx context.Context, targetFilename string, id string, getKeyFunc func(string) string) error { +func (s *SObjectBackupStorage) restoreObject(ctx context.Context, targetFilename string, id string, backupFilePath string, getKeyFunc func(string, string) string) error { bucket, err := s.getBucket() if err != nil { return errors.Wrap(err, "getBucket") } - reader, err := bucket.GetObject(ctx, getKeyFunc(id), nil) + reader, err := bucket.GetObject(ctx, getKeyFunc(id, backupFilePath), nil) if err != nil { return errors.Wrap(err, "GetObject") } @@ -187,47 +190,47 @@ func (s *SObjectBackupStorage) restoreObject(ctx context.Context, targetFilename return nil } -func (s *SObjectBackupStorage) RemoveBackup(ctx context.Context, backupId string) error { - return s.removeObject(ctx, backupId, s.getBackupKey) +func (s *SObjectBackupStorage) RemoveBackup(ctx context.Context, backupId string, backupFilePath string) error { + return s.removeObject(ctx, backupId, backupFilePath, s.getBackupKey) } func (s *SObjectBackupStorage) RemoveBackupInstance(ctx context.Context, backupId string) error { - return s.removeObject(ctx, backupId, s.getBackupInstanceKey) + return s.removeObject(ctx, backupId, "", s.getBackupInstanceKey) } -func (s *SObjectBackupStorage) removeObject(ctx context.Context, id string, getKeyFunc func(string) string) error { +func (s *SObjectBackupStorage) removeObject(ctx context.Context, id string, backupFilePath string, getKeyFunc func(string, string) string) error { bucket, err := s.getBucket() if err != nil { return errors.Wrap(err, "getBucket") } - err = bucket.DeleteObject(ctx, getKeyFunc(id)) + err = bucket.DeleteObject(ctx, getKeyFunc(id, backupFilePath)) if err != nil { return errors.Wrap(err, "DeleteObject") } return nil } -func (s *SObjectBackupStorage) IsBackupExists(backupId string) (bool, string, error) { - return s.isObjectExists(backupId, s.getBackupKey) +func (s *SObjectBackupStorage) IsBackupExists(backupId string, backupFilePath string) (bool, int64, string, error) { + return s.isObjectExists(backupId, s.getBackupKey, backupFilePath) } -func (s *SObjectBackupStorage) IsBackupInstanceExists(backupId string) (bool, string, error) { - return s.isObjectExists(backupId, s.getBackupInstanceKey) +func (s *SObjectBackupStorage) IsBackupInstanceExists(backupId string) (bool, int64, string, error) { + return s.isObjectExists(backupId, s.getBackupInstanceKey, "") } -func (s *SObjectBackupStorage) isObjectExists(id string, getKeyFunc func(string) string) (bool, string, error) { +func (s *SObjectBackupStorage) isObjectExists(id string, getKeyFunc func(string, string) string, backupFilePath string) (bool, int64, string, error) { bucket, err := s.getBucket() if err != nil { - return false, "", errors.Wrap(err, "getBucket") + return false, -1, "", errors.Wrap(err, "getBucket") } - _, err = cloudprovider.GetIObject(bucket, getKeyFunc(id)) + obj, err := cloudprovider.GetIObject(bucket, getKeyFunc(id, backupFilePath)) if err != nil { if errors.Cause(err) == errors.ErrNotFound { - return false, "", nil + return false, -1, "", nil } - return false, "", errors.Wrap(err, "GetIObject") + return false, -1, "", errors.Wrap(err, "GetIObject") } - return true, "", nil + return true, obj.GetSizeBytes(), "", nil } func (s *SObjectBackupStorage) IsOnline() (bool, string, error) { @@ -238,7 +241,7 @@ func (s *SObjectBackupStorage) IsOnline() (bool, string, error) { return exist, "", nil } -func (s *SObjectBackupStorage) GetExternalAccessUrl(backupId string) (string, error) { +func (s *SObjectBackupStorage) GetExternalAccessUrl(backupId string, backupFilePath string) (string, error) { var bucket cloudprovider.ICloudBucket var err error bucket, err = s.getExtBucket() @@ -249,7 +252,7 @@ func (s *SObjectBackupStorage) GetExternalAccessUrl(backupId string) (string, er return "", errors.Wrap(err, "getBucket") } } - url, err := bucket.GetTempUrl(http.MethodGet, s.getBackupKey(backupId), 6*time.Hour) + url, err := bucket.GetTempUrl(http.MethodGet, s.getBackupKey(backupId, backupFilePath), 6*time.Hour) if err != nil { return "", errors.Wrap(err, "GetTempUrl") } diff --git a/pkg/hostman/storageman/diskhandlers/diskhandler.go b/pkg/hostman/storageman/diskhandlers/diskhandler.go index a4bebc39ac..036e12c58f 100644 --- a/pkg/hostman/storageman/diskhandlers/diskhandler.go +++ b/pkg/hostman/storageman/diskhandlers/diskhandler.go @@ -446,7 +446,7 @@ func diskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, storag return nil, nil } -func diskStorageBackupRecovery(ctx context.Context, storage storageman.IStorage, diskId string, disk storageman.IDisk, body jsonutils.JSONObject) (interface{}, error) { +/*func diskStorageBackupRecovery(ctx context.Context, storage storageman.IStorage, diskId string, disk storageman.IDisk, body jsonutils.JSONObject) (interface{}, error) { backupId, err := body.GetString("backup_id") if err != nil { return nil, httperrors.NewMissingParameterError("backup_id") @@ -465,7 +465,7 @@ func diskStorageBackupRecovery(ctx context.Context, storage storageman.IStorage, BackupStorageAccessInfo: backupStorageAccessInfo.(*jsonutils.JSONDict), }) return nil, nil -} +}*/ func diskBackup(ctx context.Context, userCred mcclient.TokenCredential, storage storageman.IStorage, diskId string, disk storageman.IDisk, body jsonutils.JSONObject) (interface{}, error) { backupInfo := &storageman.SDiskBackup{} diff --git a/pkg/hostman/storageman/storage_local.go b/pkg/hostman/storageman/storage_local.go index 0f4ca4d0bf..f6541461a7 100644 --- a/pkg/hostman/storageman/storage_local.go +++ b/pkg/hostman/storageman/storage_local.go @@ -41,7 +41,6 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" "yunion.io/x/onecloud/pkg/hostman/options" - "yunion.io/x/onecloud/pkg/hostman/storageman/backupstorage" "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient/auth" @@ -173,19 +172,19 @@ func (s *SLocalStorage) CreateDiskFromBackup(ctx context.Context, disk IDisk, in return nil, nil }*/ -func (s *SLocalStorage) storageBackupRecovery(ctx context.Context, sbParams *SStorageBackup) (jsonutils.JSONObject, error) { +/*func (s *SLocalStorage) storageBackupRecovery(ctx context.Context, sbParams *SStorageBackup) (jsonutils.JSONObject, error) { backupStorage, err := backupstorage.GetBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo) if err != nil { return nil, err } backupPath := path.Join(s.GetBackupDir(), sbParams.BackupId) - return nil, backupStorage.RestoreBackupTo(ctx, backupPath, sbParams.BackupId) + return nil, backupStorage.RestoreBackupTo(ctx, backupPath, sbParams.BackupId, sbParams.BackupFilePath) } func (s *SLocalStorage) StorageBackupRecovery(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { sbParams := params.(*SStorageBackup) return s.storageBackupRecovery(ctx, sbParams) -} +}*/ func (s *SLocalStorage) GetAvailSizeMb() int { sizeMb := s.SBaseStorage.GetAvailSizeMb() diff --git a/pkg/hostman/storageman/storage_lvm.go b/pkg/hostman/storageman/storage_lvm.go index 478ba9480d..77ad7ee5f6 100644 --- a/pkg/hostman/storageman/storage_lvm.go +++ b/pkg/hostman/storageman/storage_lvm.go @@ -571,7 +571,7 @@ func (s *SLVMStorage) StorageBackup(ctx context.Context, params *SStorageBackup) return nil, err } backupPath := params.BackupLocalPath - err = backupstorage.SaveBackupFromFile(ctx, backupPath, params.BackupId, backupStorage) + err = backupstorage.SaveBackupFromFile(ctx, backupPath, params.BackupId, params.BackupFilePath, backupStorage) if err != nil { return nil, err } diff --git a/pkg/hostman/storageman/storagehandler/storagehandler.go b/pkg/hostman/storageman/storagehandler/storagehandler.go index cebb034e4b..15b48bbc97 100644 --- a/pkg/hostman/storageman/storagehandler/storagehandler.go +++ b/pkg/hostman/storageman/storagehandler/storagehandler.go @@ -270,12 +270,19 @@ func storageSyncBackup(ctx context.Context, w http.ResponseWriter, r *http.Reque hostutils.Response(ctx, w, httperrors.NewMissingParameterError("backup_storage_access_info")) return } - backupStorage, err := backupstorage.GetBackupStorage(backupStorageId, backupStorageAccessInfo.(*jsonutils.JSONDict)) + accessInfo := compute.SBackupStorageAccessInfo{} + err = backupStorageAccessInfo.Unmarshal(&accessInfo) + if err != nil { + hostutils.Response(ctx, w, httperrors.NewInputParameterError("unmarshal backup_storage_access_info failed %s", err)) + return + } + backupStorage, err := backupstorage.GetBackupStorage(backupStorageId, &accessInfo) if err != nil { hostutils.Response(ctx, w, err) return } - exist, reason, err := backupStorage.IsBackupExists(backupId) + backupFilePath, _ := body.GetString("backup_file_path") + exist, _, reason, err := backupStorage.IsBackupExists(backupId, backupFilePath) if err != nil { hostutils.Response(ctx, w, err) return @@ -295,6 +302,7 @@ func storageSyncBackup(ctx context.Context, w http.ResponseWriter, r *http.Reque } } ret.Set("status", jsonutils.NewString(status)) + ret.Set("reason", jsonutils.NewString(reason)) hostutils.Response(ctx, w, ret) } @@ -310,7 +318,13 @@ func storageSyncBackupStorage(ctx context.Context, w http.ResponseWriter, r *htt hostutils.Response(ctx, w, httperrors.NewMissingParameterError("backup_storage_access_info")) return } - backupStorage, err := backupstorage.GetBackupStorage(backupStorageId, backupStorageAccessInfo.(*jsonutils.JSONDict)) + accessInfo := compute.SBackupStorageAccessInfo{} + err = backupStorageAccessInfo.Unmarshal(&accessInfo) + if err != nil { + hostutils.Response(ctx, w, httperrors.NewInputParameterError("unmarshal backup_storage_access_info failed %s", err)) + return + } + backupStorage, err := backupstorage.GetBackupStorage(backupStorageId, &accessInfo) if err != nil { hostutils.Response(ctx, w, err) return @@ -339,7 +353,7 @@ func storagePackInstanceBackup(ctx context.Context, w http.ResponseWriter, r *ht if !checkOptions(ctx, w, body, "package_name", "backup_ids", "backup_storage_id", "backup_storage_access_info", "metadata") { return } - pb := storageman.SStoragePackInstanceBackup{} + pb := compute.SStoragePackInstanceBackup{} err := body.Unmarshal(&pb) if err != nil { hostutils.Response(ctx, w, httperrors.NewInputParameterError("%s", err.Error())) @@ -367,7 +381,7 @@ func storageUnpackInstanceBackup(ctx context.Context, w http.ResponseWriter, r * } func packInstanceBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { - sbParams := params.(*storageman.SStoragePackInstanceBackup) + sbParams := params.(*compute.SStoragePackInstanceBackup) packFileName, err := storageman.DoInstancePackBackup(ctx, *sbParams) if err != nil { return nil, errors.Wrap(err, "DoInstancePackBackup") @@ -410,10 +424,18 @@ func storageDeleteBackup(ctx context.Context, w http.ResponseWriter, r *http.Req hostutils.Response(ctx, w, httperrors.NewMissingParameterError("backup_storage_access_info")) return } + accessInfo := compute.SBackupStorageAccessInfo{} + err = backupStorageAccessInfo.Unmarshal(&accessInfo) + if err != nil { + hostutils.Response(ctx, w, httperrors.NewInputParameterError("unmarshal backup_storage_access_info failed %s", err)) + return + } + backupFilePath, _ := body.GetString("backup_file_path") hostutils.DelayTask(ctx, deleteBackup, &storageman.SStorageBackup{ BackupId: backupId, BackupStorageId: backupStorageId, - BackupStorageAccessInfo: backupStorageAccessInfo.(*jsonutils.JSONDict), + BackupStorageAccessInfo: &accessInfo, + BackupFilePath: backupFilePath, }) hostutils.ResponseOk(ctx, w) } @@ -435,7 +457,7 @@ func deleteBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject if err != nil { return nil, err } - err = backupStorage.RemoveBackup(ctx, sbParams.BackupId) + err = backupStorage.RemoveBackup(ctx, sbParams.BackupId, sbParams.BackupFilePath) if err != nil { return nil, err } diff --git a/pkg/hostman/storageman/storagehelper.go b/pkg/hostman/storageman/storagehelper.go index b03db974d7..17ed323019 100644 --- a/pkg/hostman/storageman/storagehelper.go +++ b/pkg/hostman/storageman/storagehelper.go @@ -77,44 +77,39 @@ type SStorageDeleteSnapshot struct { } type SDiskBackup struct { - SnapshotId string `json:"snapshot_id"` - SnapshotLocation string `json:"snapshot_location"` - BackupId string `json:"backup_id"` - BackupStorageId string `json:"backup_storage_id"` - BackupStorageAccessInfo *jsonutils.JSONDict `json:"backup_storage_access_info"` + SnapshotId string `json:"snapshot_id"` + SnapshotLocation string `json:"snapshot_location"` + BackupId string `json:"backup_id"` + BackupStorageId string `json:"backup_storage_id"` + BackupStorageAccessInfo *api.SBackupStorageAccessInfo `json:"backup_storage_access_info"` EncryptKeyId string `json:"encrypt_key_id"` UserCred mcclient.TokenCredential + + BackupFilePath string `json:"backup_file_path"` } type SStorageBackup struct { BackupId string BackupLocalPath string BackupStorageId string - BackupStorageAccessInfo *jsonutils.JSONDict + BackupStorageAccessInfo *api.SBackupStorageAccessInfo + BackupFilePath string } type SStoragePackBackup struct { PackageName string BackupId string BackupStorageId string - BackupStorageAccessInfo *jsonutils.JSONDict + BackupStorageAccessInfo *api.SBackupStorageAccessInfo Metadata api.DiskBackupPackMetadata } -type SStoragePackInstanceBackup struct { - PackageName string - BackupStorageId string - BackupStorageAccessInfo *jsonutils.JSONDict - BackupIds []string - Metadata api.InstanceBackupPackMetadata -} - type SStorageUnpackInstanceBackup struct { PackageName string BackupStorageId string - BackupStorageAccessInfo *jsonutils.JSONDict + BackupStorageAccessInfo *api.SBackupStorageAccessInfo MetadataOnly *bool } diff --git a/pkg/mcclient/options/compute/backup.go b/pkg/mcclient/options/compute/backup.go index 8176795c8b..d95cad425c 100644 --- a/pkg/mcclient/options/compute/backup.go +++ b/pkg/mcclient/options/compute/backup.go @@ -67,6 +67,8 @@ type DiskBackupCreateOptions struct { DISKID string `help:"disk id" json:"disk_id"` BACKUPSTORAGEID string `help:"backup storage id" json:"backup_storage_id"` + + BackupPath string `help:"backup path" json:"backup_path"` } func (opts *DiskBackupCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -92,6 +94,9 @@ func (opts *DiskBackupCreateOptions) Params() (jsonutils.JSONObject, error) { if opts.AsTarIgnoreNotExistFile { input.BackupAsTar.IgnoreNotExistFile = opts.AsTarIgnoreNotExistFile } + if opts.BackupPath != "" { + input.BackupFilePath = opts.BackupPath + } return jsonutils.Marshal(input), nil }