feat(region,host): support packing and unpacking instancebackup

This commit is contained in:
rainzm
2022-02-23 10:04:07 +08:00
parent 9ce60bb405
commit 77ea86fafe
20 changed files with 826 additions and 13 deletions
+2
View File
@@ -43,4 +43,6 @@ func init() {
ibCmd.Show(&compute.InstanceBackupIdOptions{})
ibCmd.Delete(&compute.InstanceBackupIdOptions{})
ibCmd.Perform("recovery", &compute.InstanceBackupRecoveryOptions{})
ibCmd.Perform("pack", &compute.InstanceBackupPackOptions{})
ibCmd.PerformClass("create-from-package", &compute.InstanceBackupManagerCreateFromPackageOptions{})
}
+27 -1
View File
@@ -14,7 +14,11 @@
package compute
import "yunion.io/x/onecloud/pkg/apis"
import (
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/apis"
)
const (
BACKUPSTORAGE_TYPE_NFS = "nfs"
@@ -119,3 +123,25 @@ type DiskBackupRecoveryInput struct {
type DiskBackupSyncstatusInput struct {
}
type DiskBackupPackMetadata struct {
OsArch string
SizeMb int
DiskSizeMb int
DiskType string
// 操作系统类型
OsType string
DiskConfig *SBackupDiskConfig
}
type InstanceBackupPackMetadata struct {
OsArch string
ServerConfig jsonutils.JSONObject
ServerMetadata jsonutils.JSONObject
SecGroups jsonutils.JSONObject
KeypairId string
OsType string
InstanceType string
SizeMb int
DiskMetadatas []DiskBackupPackMetadata
}
+15
View File
@@ -10,6 +10,11 @@ const (
INSTANCE_BACKUP_STATUS_RECOVERY = "recovery"
INSTANCE_BACKUP_STATUS_RECOVERY_FAILED = "recovery_failed"
INSTANCE_BACKUP_STATUS_READY = "ready"
INSTANCE_BACKUP_STATUS_PACK = "pack"
INSTANCE_BACKUP_STATUS_PACK_FAILED = "pack_failed"
INSTANCE_BACKUP_STATUS_CREATING_FROM_PACKAGE = "creating_from_package"
INSTANCE_BACKUP_STATUS_CREATE_FROM_PACKAGE_FAILED = "create_from_package_failed"
)
type InstanceBackupListInput struct {
@@ -44,3 +49,13 @@ type InstanceBackupRecoveryInput struct {
// description: name of guest
Name string
}
type InstanceBackupPackInput struct {
PackageName string
}
type InstanceBackupManagerCreateFromPackageInput struct {
BackupStorageId string
PackageName string
Name string
}
+7
View File
@@ -287,4 +287,11 @@ const (
ACT_MERGE_NETWORK = "merge_network"
ACT_MERGE_NETWORK_FAILED = "merge_network_failed"
ACT_RECOVERY = "recovery"
ACT_RECOVERY_FAIL = "recovery_fail"
ACT_PACK = "pack"
ACT_PACK_FAIL = "pack_fail"
ACT_UNPACK = "unpack"
ACT_UNPACK_FAIL = "unpack_fail"
)
+41
View File
@@ -437,3 +437,44 @@ func (self *SDiskBackup) PerformSyncstatus(ctx context.Context, userCred mcclien
return nil, StartResourceSyncStatusTask(ctx, userCred, self, "DiskBackupSyncstatusTask", "")
}
func (self *SDiskBackup) PackMetadata() *api.DiskBackupPackMetadata {
return &api.DiskBackupPackMetadata{
OsArch: self.OsArch,
SizeMb: self.SizeMb,
DiskSizeMb: self.DiskSizeMb,
DiskType: self.DiskType,
// 操作系统类型
OsType: self.OsType,
DiskConfig: &api.SBackupDiskConfig{
DiskConfig: self.DiskConfig.DiskConfig,
Name: self.DiskConfig.Name,
},
}
}
func (manager *SDiskBackupManager) CreateFromPackMetadata(ctx context.Context, owner mcclient.TokenCredential, backupStorageId, id, name string, metadata *api.DiskBackupPackMetadata) (*SDiskBackup, error) {
backup := &SDiskBackup{}
backup.SetModelManager(manager, backup)
backup.ProjectId = owner.GetProjectId()
backup.DomainId = owner.GetProjectDomainId()
backup.DiskConfig = &SBackupDiskConfig{
DiskConfig: metadata.DiskConfig.DiskConfig,
Name: metadata.DiskConfig.Name,
}
backup.DiskType = metadata.DiskType
backup.DiskSizeMb = metadata.DiskSizeMb
backup.OsArch = metadata.OsArch
backup.DiskType = metadata.DiskType
backup.OsType = metadata.OsType
backup.CloudregionId = "default"
backup.BackupStorageId = backupStorageId
backup.Name = name
backup.Id = id
backup.Status = api.BACKUP_STATUS_READY
err := DiskBackupManager.TableSpec().Insert(ctx, backup)
if err != nil {
return nil, err
}
return backup, nil
}
+113
View File
@@ -17,6 +17,7 @@ package models
import (
"context"
"database/sql"
"fmt"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
@@ -436,3 +437,115 @@ func (self *SInstanceBackup) StartRecoveryTask(ctx context.Context, userCred mcc
}
return nil
}
func (self *SInstanceBackup) PerformPack(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.InstanceBackupPackInput) (jsonutils.JSONObject, error) {
if input.PackageName == "" {
return nil, httperrors.NewMissingParameterError("miss package_name")
}
self.SetStatus(userCred, api.INSTANCE_BACKUP_STATUS_PACK, "")
params := jsonutils.NewDict()
params.Set("package_name", jsonutils.NewString(input.PackageName))
task, err := taskman.TaskManager.NewTask(ctx, "InstanceBackupPackTask", self, userCred, params, "", "", nil)
if err != nil {
return nil, err
} else {
task.ScheduleRun(nil)
}
return nil, nil
}
func (self *SInstanceBackupManager) PerformCreateFromPackage(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.InstanceBackupManagerCreateFromPackageInput) (jsonutils.JSONObject, error) {
if input.Name == "" {
return nil, httperrors.NewMissingParameterError("miss name")
}
if input.PackageName == "" {
return nil, httperrors.NewMissingParameterError("miss package_name")
}
_, err := BackupStorageManager.FetchById(input.BackupStorageId)
if err != nil {
return nil, httperrors.NewInputParameterError("unable to fetch backupStorage %s", input.BackupStorageId)
}
name, err := db.GenerateName(ctx, self, userCred, input.Name)
if err != nil {
return nil, errors.Wrap(err, "unable to generate name")
}
ib, err := self.CreateInstanceBackupFromPackage(ctx, userCred, input.BackupStorageId, name)
if err != nil {
return nil, errors.Wrap(err, "unable to create instanceBackup")
}
params := jsonutils.NewDict()
params.Set("package_name", jsonutils.NewString(input.PackageName))
task, err := taskman.TaskManager.NewTask(ctx, "InstanceBackupUnpackTask", ib, userCred, params, "", "", nil)
if err != nil {
return nil, err
} else {
task.ScheduleRun(nil)
}
return nil, nil
}
func (self *SInstanceBackup) PackMetadata() (*api.InstanceBackupPackMetadata, error) {
metadata := &api.InstanceBackupPackMetadata{
OsArch: self.OsArch,
ServerConfig: self.ServerConfig,
ServerMetadata: self.ServerMetadata,
SecGroups: self.SecGroups,
KeypairId: self.KeypairId,
OsType: self.OsType,
InstanceType: self.InstanceType,
SizeMb: self.SizeMb,
}
dbs, err := self.GetBackups()
if err != nil {
return nil, err
}
for i := range dbs {
mt := dbs[i].PackMetadata()
metadata.DiskMetadatas = append(metadata.DiskMetadatas, *mt)
}
return metadata, nil
}
func (manager *SInstanceBackupManager) CreateInstanceBackupFromPackage(ctx context.Context, owner mcclient.TokenCredential, backupStorageId, name string) (*SInstanceBackup, error) {
ib := &SInstanceBackup{}
ib.SetModelManager(manager, ib)
ib.ProjectId = owner.GetProjectId()
ib.DomainId = owner.GetProjectDomainId()
ib.Name = name
ib.BackupStorageId = backupStorageId
ib.CloudregionId = "default"
ib.Status = api.INSTANCE_BACKUP_STATUS_CREATING_FROM_PACKAGE
err := manager.TableSpec().Insert(ctx, ib)
if err != nil {
return nil, errors.Wrap(err, "unable to insert instance backup")
}
return ib, nil
}
func (ib *SInstanceBackup) FillFromPackMetadata(ctx context.Context, userCred mcclient.TokenCredential, diskBackupIds []string, metadata *api.InstanceBackupPackMetadata) (*SInstanceBackup, error) {
for i, backupId := range diskBackupIds {
_, err := DiskBackupManager.CreateFromPackMetadata(ctx, userCred, ib.BackupStorageId, backupId, fmt.Sprintf("%s_disk_%d", ib.Name, i), &metadata.DiskMetadatas[i])
if err != nil {
return nil, errors.Wrapf(err, "unable to create diskbackup %s", backupId)
}
err = InstanceBackupJointManager.CreateJoint(ctx, ib.GetId(), backupId, int8(i))
if err != nil {
return nil, errors.Wrapf(err, "unable to CreateJoint for instanceBackup %s and diskBackup %s", ib.GetId(), backupId)
}
}
_, err := db.Update(ib, func() error {
ib.OsArch = metadata.OsArch
ib.ServerConfig = metadata.ServerConfig
ib.ServerMetadata = metadata.ServerMetadata
ib.SecGroups = metadata.SecGroups
ib.KeypairId = metadata.KeypairId
ib.OsType = metadata.OsType
ib.InstanceType = metadata.InstanceType
ib.SizeMb = metadata.SizeMb
return nil
})
if err != nil {
return nil, err
}
return ib, nil
}
+2
View File
@@ -126,6 +126,8 @@ type IRegionDriver interface {
RequestCreateInstanceSnapshot(ctx context.Context, guest *SGuest, isp *SInstanceSnapshot, task taskman.ITask, params *jsonutils.JSONDict) error
RequestDeleteInstanceSnapshot(ctx context.Context, isp *SInstanceSnapshot, task taskman.ITask) error
RequestResetToInstanceSnapshot(ctx context.Context, guest *SGuest, isp *SInstanceSnapshot, task taskman.ITask, params *jsonutils.JSONDict) error
RequestPackInstanceBackup(ctx context.Context, ib *SInstanceBackup, task taskman.ITask, packageName string) error
RequestUnpackInstanceBackup(ctx context.Context, ib *SInstanceBackup, task taskman.ITask, packageName string) error
RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, secgroup *SSecurityGroup, classic bool, removeProjectId string, task taskman.ITask) error
RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *SVpc, secgroup *SSecurityGroup, removeProjectId, service string) (string, error)
+7
View File
@@ -492,3 +492,10 @@ func (self *SBaseRegionDriver) RequestDeleteInstanceBackup(ctx context.Context,
func (self *SBaseRegionDriver) ValidateCreateCdnData(ctx context.Context, userCred mcclient.TokenCredential, input api.CDNDomainCreateInput) (api.CDNDomainCreateInput, error) {
return input, errors.Wrapf(cloudprovider.ErrNotImplemented, "ValidateCreateCdnData")
}
func (self *SBaseRegionDriver) RequestPackInstanceBackup(ctx context.Context, ib *models.SInstanceBackup, task taskman.ITask, packageName string) error {
return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestPackInstanceBackup")
}
func (self *SBaseRegionDriver) RequestUnpackInstanceBackup(ctx context.Context, ib *models.SInstanceBackup, task taskman.ITask, packageName string) error {
return errors.Wrapf(cloudprovider.ErrNotImplemented, "RequestUnpackInstanceBackup")
}
+67
View File
@@ -1296,6 +1296,73 @@ func (self *SKVMRegionDriver) RequestCreateInstanceBackup(ctx context.Context, g
return nil
}
func (self *SKVMRegionDriver) RequestPackInstanceBackup(ctx context.Context, ib *models.SInstanceBackup, task taskman.ITask, packageName string) error {
backupStorage, err := ib.GetBackupStorage()
if err != nil {
return errors.Wrap(err, "unable to get backupStorage")
}
backups, err := ib.GetBackups()
if err != nil {
return errors.Wrap(err, "unable to get backups")
}
storage, _ := backups[0].GetStorage()
var host *models.SHost
if storage != nil {
host = storage.GetMasterHost()
}
if host == nil {
host, err = models.HostManager.GetEnabledKvmHost()
if err != nil {
return errors.Wrap(err, "unable to GetEnabledKvmHost")
}
}
backupIds := make([]string, len(backups))
for i := range backupIds {
backupIds[i] = backups[i].GetId()
}
metadata, err := ib.PackMetadata()
if err != nil {
return errors.Wrap(err, "unable to PackMetadata")
}
url := fmt.Sprintf("%s/storages/pack-instance-backup", host.ManagerUri)
body := jsonutils.NewDict()
body.Set("package_name", jsonutils.NewString(packageName))
body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId()))
body.Set("backup_storage_access_info", jsonutils.Marshal(backupStorage.AccessInfo))
body.Set("backup_ids", jsonutils.Marshal(backupIds))
body.Set("metadata", jsonutils.Marshal(metadata))
header := task.GetTaskRequestHeader()
_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false)
if err != nil {
return errors.Wrap(err, "unable to pack instancebackup")
}
return nil
}
func (self *SKVMRegionDriver) RequestUnpackInstanceBackup(ctx context.Context, ib *models.SInstanceBackup, task taskman.ITask, packageName string) error {
log.Infof("RequestUnpackInstanceBackup")
backupStorage, err := ib.GetBackupStorage()
if err != nil {
return errors.Wrap(err, "unable to get backupStorage")
}
host, err := models.HostManager.GetEnabledKvmHost()
if err != nil {
return errors.Wrap(err, "unable to GetEnabledKvmHost")
}
url := fmt.Sprintf("%s/storages/unpack-instance-backup", host.ManagerUri)
log.Infof("url: %s", url)
body := jsonutils.NewDict()
body.Set("package_name", jsonutils.NewString(packageName))
body.Set("backup_storage_id", jsonutils.NewString(backupStorage.GetId()))
body.Set("backup_storage_access_info", jsonutils.Marshal(backupStorage.AccessInfo))
header := task.GetTaskRequestHeader()
_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false)
if err != nil {
return errors.Wrap(err, "unable to pack instancebackup")
}
return nil
}
func (self *SKVMRegionDriver) RequestSyncDiskBackupStatus(ctx context.Context, userCred mcclient.TokenCredential, backup *models.SDiskBackup, task taskman.ITask) error {
taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) {
originStatus, _ := task.GetParams().GetString("origin_status")
@@ -0,0 +1,69 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/util/logclient"
)
type InstanceBackupPackTask struct {
taskman.STask
}
func init() {
taskman.RegisterTask(InstanceBackupPackTask{})
}
func (self *InstanceBackupPackTask) taskFailed(ctx context.Context, ib *models.SInstanceBackup, reason jsonutils.JSONObject) {
reasonStr, _ := reason.GetString()
ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_PACK_FAILED, reasonStr)
logclient.AddActionLogWithStartable(self, ib, logclient.ACT_PACK, reason, self.UserCred, false)
db.OpsLog.LogEvent(ib, db.ACT_PACK_FAIL, ib.GetShortDesc(ctx), self.GetUserCred())
self.SetStageFailed(ctx, reason)
}
func (self *InstanceBackupPackTask) taskSuccess(ctx context.Context, ib *models.SInstanceBackup) {
ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_READY, "")
logclient.AddActionLogWithStartable(self, ib, logclient.ACT_PACK, nil, self.UserCred, true)
db.OpsLog.LogEvent(ib, db.ACT_PACK, ib.GetShortDesc(ctx), self.GetUserCred())
self.SetStageComplete(ctx, nil)
}
func (self *InstanceBackupPackTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
ib := obj.(*models.SInstanceBackup)
packageName, _ := self.GetParams().GetString("package_name")
self.SetStage("OnPackComplete", nil)
err := ib.GetRegionDriver().RequestPackInstanceBackup(ctx, ib, self, packageName)
if err != nil {
self.taskFailed(ctx, ib, jsonutils.NewString(err.Error()))
return
}
}
func (self *InstanceBackupPackTask) OnPackComplete(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
self.taskSuccess(ctx, ib)
}
func (self *InstanceBackupPackTask) OnPackCompleteFailed(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
self.taskFailed(ctx, ib, data)
}
@@ -42,12 +42,14 @@ func (self *InstanceBackupRecoveryTask) taskFailed(ctx context.Context, ib *mode
reasonStr, _ := reason.GetString()
ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_CREATE_FAILED, reasonStr)
logclient.AddActionLogWithStartable(self, ib, logclient.ACT_RECOVERY, reason, self.UserCred, false)
db.OpsLog.LogEvent(ib, db.ACT_RECOVERY_FAIL, ib.GetShortDesc(ctx), self.GetUserCred())
self.SetStageFailed(ctx, reason)
}
func (self *InstanceBackupRecoveryTask) taskSuccess(ctx context.Context, ib *models.SInstanceBackup) {
ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_READY, "")
logclient.AddActionLogWithStartable(self, ib, logclient.ACT_RECOVERY, nil, self.UserCred, true)
db.OpsLog.LogEvent(ib, db.ACT_RECOVERY, ib.GetShortDesc(ctx), self.GetUserCred())
self.SetStageComplete(ctx, nil)
}
@@ -0,0 +1,86 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks
import (
"context"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/util/logclient"
)
type InstanceBackupUnpackTask struct {
taskman.STask
}
func init() {
taskman.RegisterTask(InstanceBackupUnpackTask{})
}
func (self *InstanceBackupUnpackTask) taskFailed(ctx context.Context, ib *models.SInstanceBackup, reason jsonutils.JSONObject) {
reasonStr, _ := reason.GetString()
ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_CREATE_FROM_PACKAGE_FAILED, reasonStr)
logclient.AddActionLogWithStartable(self, ib, logclient.ACT_UNPACK, reason, self.UserCred, false)
db.OpsLog.LogEvent(ib, db.ACT_UNPACK_FAIL, ib.GetShortDesc(ctx), self.GetUserCred())
self.SetStageFailed(ctx, reason)
}
func (self *InstanceBackupUnpackTask) taskSuccess(ctx context.Context, ib *models.SInstanceBackup) {
ib.SetStatus(self.UserCred, compute.INSTANCE_BACKUP_STATUS_READY, "")
logclient.AddActionLogWithStartable(self, ib, logclient.ACT_UNPACK, nil, self.UserCred, true)
db.OpsLog.LogEvent(ib, db.ACT_UNPACK, ib.GetShortDesc(ctx), self.GetUserCred())
self.SetStageComplete(ctx, nil)
}
func (self *InstanceBackupUnpackTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
ib := obj.(*models.SInstanceBackup)
packageName, _ := self.GetParams().GetString("package_name")
self.SetStage("OnUnpackComplete", nil)
err := ib.GetRegionDriver().RequestUnpackInstanceBackup(ctx, ib, self, packageName)
if err != nil {
self.taskFailed(ctx, ib, jsonutils.NewString(err.Error()))
return
}
}
func (self *InstanceBackupUnpackTask) OnUnpackComplete(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
diskBackupIds := make([]string, 0)
err := data.Unmarshal(&diskBackupIds, "disk_backup_ids")
if err != nil {
self.taskFailed(ctx, ib, jsonutils.NewString(err.Error()))
return
}
metadata := &compute.InstanceBackupPackMetadata{}
err = data.Unmarshal(metadata, "metadata")
if err != nil {
self.taskFailed(ctx, ib, jsonutils.NewString(err.Error()))
return
}
_, err = ib.FillFromPackMetadata(ctx, self.GetUserCred(), diskBackupIds, metadata)
if err != nil {
self.taskFailed(ctx, ib, jsonutils.NewString(err.Error()))
return
}
self.taskSuccess(ctx, ib)
}
func (self *InstanceBackupUnpackTask) OnUnpackCompleteFailed(ctx context.Context, ib *models.SInstanceBackup, data jsonutils.JSONObject) {
self.taskFailed(ctx, ib, data)
}
@@ -16,9 +16,11 @@ package backupstorage
import (
"fmt"
"sync"
"yunion.io/x/jsonutils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/util/qemuimg"
)
@@ -29,8 +31,12 @@ type IBackupStorage interface {
IsExists(backupId string) (bool, error)
ConvertTo(destPath string, format qemuimg.TImageFormat, backupId string) error
ConvertFrom(srcPath string, format qemuimg.TImageFormat, backupId string) (int, error)
InstancePack(packageName string, backupIds []string, metadata *api.InstanceBackupPackMetadata) error
InstanceUnpack(packageName string) ([]string, *api.InstanceBackupPackMetadata, error)
}
var backupStoragePool *sync.Map = &sync.Map{}
func NewBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) {
nfsHost, err := backupStorageAccessInfo.GetString("nfs_host")
if err != nil {
@@ -42,3 +48,12 @@ func NewBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils
}
return NewNFSBackupStorage(backupStroageId, nfsHost, nfsSharedDir), nil
}
func GetBackupStorage(backupStroageId string, backupStorageAccessInfo *jsonutils.JSONDict) (IBackupStorage, error) {
bs, err := NewBackupStorage(backupStroageId, backupStorageAccessInfo)
if err != nil {
return nil, err
}
ibs, _ := backupStoragePool.LoadOrStore(backupStroageId, bs)
return ibs.(IBackupStorage), nil
}
@@ -17,12 +17,17 @@ package backupstorage
import (
"context"
"fmt"
"io/ioutil"
"path"
"sync"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/util/fileutils2"
@@ -37,6 +42,8 @@ type SNFSBackupStorage struct {
Path string
NfsHost string
NfsSharedDir string
lock *sync.Mutex
userNumber int
}
func NewNFSBackupStorage(backupStorageId, nfsHost, nfsSharedDir string) *SNFSBackupStorage {
@@ -45,6 +52,7 @@ func NewNFSBackupStorage(backupStorageId, nfsHost, nfsSharedDir string) *SNFSBac
NfsHost: nfsHost,
NfsSharedDir: nfsSharedDir,
Path: path.Join(BackupStoragePath, backupStorageId),
lock: &sync.Mutex{},
}
}
@@ -52,9 +60,13 @@ func (s *SNFSBackupStorage) getBackupDir() string {
return path.Join(s.Path, "backups")
}
func (s *SNFSBackupStorage) getPackageDir() string {
return path.Join(s.Path, "backuppacks")
}
func (s *SNFSBackupStorage) checkAndMount() error {
lockman.LockRawObject(context.Background(), "backupstorage", s.BackupStorageId)
defer lockman.ReleaseRawObject(context.Background(), "backupstorage", s.BackupStorageId)
s.lock.Lock()
defer s.lock.Unlock()
if !fileutils2.Exists(s.Path) {
output, err := procutils.NewCommand("mkdir", "-p", s.Path).Output()
if err != nil {
@@ -63,6 +75,7 @@ func (s *SNFSBackupStorage) checkAndMount() error {
}
}
if err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", s.Path).Run(); err == nil {
s.userNumber++
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -80,12 +93,25 @@ func (s *SNFSBackupStorage) checkAndMount() error {
return errors.Wrapf(err, "mkdir %s failed: %s", backupDir, output)
}
}
packageDir := s.getPackageDir()
if !fileutils2.Exists(packageDir) {
output, err := procutils.NewCommand("mkdir", "-p", packageDir).Output()
if err != nil {
log.Errorf("mkdir %s failed: %s", packageDir, output)
return errors.Wrapf(err, "mkdir %s failed: %s", packageDir, output)
}
}
s.userNumber++
return nil
}
func (s *SNFSBackupStorage) unMount() error {
lockman.LockRawObject(context.Background(), "backupstorage", s.BackupStorageId)
defer lockman.ReleaseRawObject(context.Background(), "backupstorage", s.BackupStorageId)
s.lock.Lock()
defer s.lock.Unlock()
s.userNumber--
if s.userNumber > 0 {
return nil
}
out, err := procutils.NewRemoteCommandAsFarAsPossible("umount", s.Path).Output()
if err != nil {
return errors.Wrapf(err, "umount %s failed %s", s.Path, out)
@@ -123,6 +149,223 @@ func (s *SNFSBackupStorage) CopyBackupTo(targetFilename string, backupId string)
return nil
}
const (
PackageDiskFilename = "disk"
PackageMetadataFilename = "metadata"
)
func (s *SNFSBackupStorage) Pack(backupId string, packageName string, metadata jsonutils.JSONObject) error {
err := s.checkAndMount()
if err != nil {
return errors.Wrap(err, "unable to checkAndMount")
}
defer s.unMount()
lockman.LockRawObject(context.Background(), "package", packageName)
defer lockman.ReleaseRawObject(context.Background(), "package", packageName)
backupDir := s.getBackupDir()
backupPath := path.Join(backupDir, backupId)
packageDir := s.getPackageDir()
packagePath := path.Join(packageDir, packageName)
packageFilename := path.Join(packageDir, packageName+".tar")
if fileutils2.Exists(packageFilename) {
return errors.Error("A package with the same name already exists")
}
if fileutils2.Exists(packagePath) {
// delete residual data
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
return errors.Wrapf(err, "rm %s failed and output is %q", packagePath, output)
}
}
output, err := procutils.NewCommand("mkdir", "-p", packagePath).Output()
if err != nil {
log.Errorf("mkdir %s failed: %s", packagePath, output)
return errors.Wrapf(err, "mkdir %s failed: %s", packageDir, output)
}
defer func() {
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
}
}()
packageDiskPath := path.Join(packagePath, PackageDiskFilename)
if output, err := procutils.NewCommand("cp", backupPath, packageDiskPath).Output(); err != nil {
log.Errorf("unable to cp %s to %s: %s", backupPath, packageDiskPath, output)
return errors.Wrapf(err, "cp %s to %s failed and output is %q", backupPath, packageDiskPath, output)
}
packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
err = ioutil.WriteFile(packageMetadataPath, []byte(metadata.PrettyString()), 0644)
if err != nil {
return errors.Wrapf(err, "unable to write to %s", packageMetadataPath)
}
// tar
if output, err := procutils.NewCommand("tar", "-cf", packageFilename, packagePath).Output(); err != nil {
log.Errorf("unable to 'tar -cf %s %s': %s", packageFilename, packagePath, output)
return errors.Wrap(err, "unable to tar")
}
return nil
}
func (s *SNFSBackupStorage) UnPack(backupId string, packageName string) (jsonutils.JSONObject, error) {
err := s.checkAndMount()
if err != nil {
return nil, errors.Wrap(err, "unable to checkAndMount")
}
defer s.unMount()
lockman.LockRawObject(context.Background(), "package", packageName)
defer lockman.ReleaseRawObject(context.Background(), "package", packageName)
backupDir := s.getBackupDir()
backupPath := path.Join(backupDir, backupId)
packageDir := s.getPackageDir()
packagePath := path.Join(packageDir, packageName)
packageFilename := path.Join(packageDir, packageName+".tar")
if !fileutils2.Exists(packageFilename) {
return nil, errors.Wrapf(err, "package %s does not exists", packageName)
}
if fileutils2.Exists(packagePath) {
// delete residual data
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
return nil, errors.Wrapf(err, "rm %s failed and output is %q", packagePath, output)
}
}
// untar
if output, err := procutils.NewCommand("tar", "-xf", packageFilename, packagePath).Output(); err != nil {
log.Errorf("unable to 'tar -xf %s %s': %s", packageFilename, packagePath, output)
return nil, errors.Wrap(err, "unable to untar")
}
defer func() {
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
}
}()
packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
metadataBytes, err := ioutil.ReadFile(packageMetadataPath)
if err != nil {
return nil, errors.Wrap(err, "unable to read metadata file")
}
metadata, err := jsonutils.ParseQueryString(string(metadataBytes))
if err != nil {
return nil, errors.Wrap(err, "unable to parse string to json")
}
packageDiskPath := path.Join(packagePath, PackageDiskFilename)
if output, err := procutils.NewCommand("mv", packageDiskPath, backupPath).Output(); err != nil {
return nil, errors.Wrapf(err, "mv %s to %s failed and output is %q", packageDiskPath, backupPath, output)
}
return metadata, nil
}
func (s *SNFSBackupStorage) InstancePack(packageName string, backupIds []string, metadata *api.InstanceBackupPackMetadata) error {
err := s.checkAndMount()
if err != nil {
return errors.Wrap(err, "unable to checkAndMount")
}
defer s.unMount()
lockman.LockRawObject(context.Background(), "package", packageName)
defer lockman.ReleaseRawObject(context.Background(), "package", packageName)
backupDir := s.getBackupDir()
packageDir := s.getPackageDir()
packagePath := path.Join(packageDir, packageName)
packageFilename := path.Join(packageDir, packageName+".tar")
if fileutils2.Exists(packageFilename) {
return errors.Error("A package with the same name already exists")
}
if fileutils2.Exists(packagePath) {
// delete residual data
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
return errors.Wrapf(err, "rm %s failed and output is %q", packagePath, output)
}
}
output, err := procutils.NewCommand("mkdir", "-p", packagePath).Output()
if err != nil {
log.Errorf("mkdir %s failed: %s", packagePath, output)
return errors.Wrapf(err, "mkdir %s failed: %s", packageDir, output)
}
defer func() {
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
}
}()
for i, backupId := range backupIds {
packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i))
backupPath := path.Join(backupDir, backupId)
if output, err := procutils.NewCommand("cp", backupPath, packageDiskPath).Output(); err != nil {
log.Errorf("unable to cp %s to %s: %s", backupPath, packageDiskPath, output)
return errors.Wrapf(err, "cp %s to %s failed and output is %q", backupPath, packageDiskPath, output)
}
}
packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
err = ioutil.WriteFile(packageMetadataPath, []byte(jsonutils.Marshal(metadata).PrettyString()), 0644)
if err != nil {
return errors.Wrapf(err, "unable to write to %s", packageMetadataPath)
}
// tar
if output, err := procutils.NewCommand("tar", "-cf", packageFilename, "-C", packageDir, packageName).Output(); err != nil {
log.Errorf("unable to 'tar -cf %s -C %s %s': %s", packageFilename, packageDir, packageName, output)
return errors.Wrap(err, "unable to tar")
}
return nil
}
func (s *SNFSBackupStorage) InstanceUnpack(packageName string) ([]string, *api.InstanceBackupPackMetadata, error) {
err := s.checkAndMount()
if err != nil {
return nil, nil, errors.Wrap(err, "unable to checkAndMount")
}
defer s.unMount()
lockman.LockRawObject(context.Background(), "package", packageName)
defer lockman.ReleaseRawObject(context.Background(), "package", packageName)
backupDir := s.getBackupDir()
packageDir := s.getPackageDir()
packagePath := path.Join(packageDir, packageName)
packageFilename := path.Join(packageDir, packageName+".tar")
if !fileutils2.Exists(packageFilename) {
return nil, nil, errors.Wrapf(err, "package %s does not exists", packageName)
}
if fileutils2.Exists(packagePath) {
// delete residual data
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
return nil, nil, errors.Wrapf(err, "rm %s failed and output is %q", packagePath, output)
}
}
// untar
if output, err := procutils.NewCommand("tar", "-xf", packageFilename, "-C", packageDir, packageName).Output(); err != nil {
log.Errorf("unable to 'tar -xf %s -C %s %s': %s", packageFilename, packageDir, packageName, output)
return nil, nil, errors.Wrap(err, "unable to untar")
}
defer func() {
if output, err := procutils.NewCommand("rm", "-rf", packagePath).Output(); err != nil {
log.Errorf("unable to rm %s: %s", packagePath, output)
}
}()
packageMetadataPath := path.Join(packagePath, PackageMetadataFilename)
metadataBytes, err := ioutil.ReadFile(packageMetadataPath)
if err != nil {
return nil, nil, errors.Wrap(err, "unable to read metadata file")
}
metadataJson, err := jsonutils.Parse(metadataBytes)
if err != nil {
return nil, nil, errors.Wrap(err, "unable to parse string to json")
}
metadata := &api.InstanceBackupPackMetadata{}
err = metadataJson.Unmarshal(metadata)
if err != nil {
return nil, nil, err
}
backupIds := make([]string, len(metadata.DiskMetadatas))
for i := 0; i < len(metadata.DiskMetadatas); i++ {
backupId := db.DefaultUUIDGenerator()
backupIds[i] = backupId
backupPath := path.Join(backupDir, backupId)
packageDiskPath := path.Join(packagePath, fmt.Sprintf("%s_%d", PackageDiskFilename, i))
if output, err := procutils.NewCommand("mv", packageDiskPath, backupPath).Output(); err != nil {
return nil, nil, errors.Wrapf(err, "mv %s to %s failed and output is %q", packageDiskPath, backupPath, output)
}
}
return backupIds, metadata, nil
}
func (s *SNFSBackupStorage) ConvertFrom(srcPath string, format qemuimg.TImageFormat, backupId string) (int, error) {
err := s.checkAndMount()
if err != nil {
+2 -2
View File
@@ -131,7 +131,7 @@ func (s *SLocalStorage) CreateDiskFromBackup(ctx context.Context, disk IDisk, in
func (s *SLocalStorage) StorageBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
sbParams := params.(*SStorageBackup)
backupStorage, err := backupstorage.NewBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
backupStorage, err := backupstorage.GetBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
if err != nil {
return nil, err
}
@@ -150,7 +150,7 @@ func (s *SLocalStorage) StorageBackup(ctx context.Context, params interface{}) (
}
func (s *SLocalStorage) storageBackupRecovery(ctx context.Context, sbParams *SStorageBackup) (jsonutils.JSONObject, error) {
backupStorage, err := backupstorage.NewBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
backupStorage, err := backupstorage.GetBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
if err != nil {
return nil, err
}
+4 -4
View File
@@ -317,9 +317,9 @@ func (s *SRbdStorage) createBackup(pool string, diskId string, snapshotId string
}
defer backupImg.Delete()
// convert backupStorage
backupStorage, err := backupstorage.NewBackupStorage(backupStorageId, backupStorageAccessInfo)
backupStorage, err := backupstorage.GetBackupStorage(backupStorageId, backupStorageAccessInfo)
if err != nil {
return 0, errors.Wrap(err, "unable to NewNFSBackupStorage")
return 0, errors.Wrap(err, "unable to GetBackupStorage")
}
srcPath := fmt.Sprintf("rbd:%s/%s%s", pool, backupName, s.getStorageConfString())
// convert
@@ -570,9 +570,9 @@ func (s *SRbdStorage) CreateDiskFromBackup(ctx context.Context, disk IDisk, inpu
backup := input.DiskInfo.Backup
pool, _ := s.StorageConf.GetString("pool")
destPath := fmt.Sprintf("rbd:%s/%s%s", pool, disk.GetId(), s.getStorageConfString())
backupStorage, err := backupstorage.NewBackupStorage(backup.BackupStorageId, backup.BackupStorageAccessInfo)
backupStorage, err := backupstorage.GetBackupStorage(backup.BackupStorageId, backup.BackupStorageAccessInfo)
if err != nil {
return errors.Wrap(err, "unable to NewNFSBackupStorage")
return errors.Wrap(err, "unable to GetBackupStorage")
}
err = backupStorage.ConvertTo(destPath, qemuimg.RAW, backup.BackupId)
if err != nil {
@@ -65,6 +65,12 @@ func AddStorageHandler(prefix string, app *appsrv.Application) {
app.AddHandler("POST",
fmt.Sprintf("%s/%s/sync-backup", prefix, keyWords),
auth.Authenticate(storageSyncBackup))
app.AddHandler("POST",
fmt.Sprintf("%s/%s/pack-instance-backup", prefix, keyWords),
auth.Authenticate(storagePackInstanceBackup))
app.AddHandler("POST",
fmt.Sprintf("%s/%s/unpack-instance-backup", prefix, keyWords),
auth.Authenticate(storageUnpackInstanceBackup))
}
}
@@ -208,7 +214,7 @@ func storageSyncBackup(ctx context.Context, w http.ResponseWriter, r *http.Reque
hostutils.Response(ctx, w, httperrors.NewMissingParameterError("backup_storage_access_info"))
return
}
backupStorage, err := backupstorage.NewBackupStorage(backupStorageId, backupStorageAccessInfo.(*jsonutils.JSONDict))
backupStorage, err := backupstorage.GetBackupStorage(backupStorageId, backupStorageAccessInfo.(*jsonutils.JSONDict))
if err != nil {
hostutils.Response(ctx, w, err)
return
@@ -234,6 +240,64 @@ func storageSyncBackup(ctx context.Context, w http.ResponseWriter, r *http.Reque
hostutils.Response(ctx, w, ret)
}
func storagePackInstanceBackup(ctx context.Context, w http.ResponseWriter, r *http.Request) {
_, _, body := appsrv.FetchEnv(ctx, w, r)
if !checkOptions(ctx, w, body, "package_name", "backup_ids", "backup_storage_id", "backup_storage_access_info", "metadata") {
return
}
pb := storageman.SStoragePackInstanceBackup{}
err := body.Unmarshal(&pb)
if err != nil {
hostutils.Response(ctx, w, httperrors.NewInputParameterError(err.Error()))
return
}
hostutils.DelayTask(ctx, packInstanceBackup, &pb)
hostutils.ResponseOk(ctx, w)
}
func storageUnpackInstanceBackup(ctx context.Context, w http.ResponseWriter, r *http.Request) {
_, _, body := appsrv.FetchEnv(ctx, w, r)
if !checkOptions(ctx, w, body, "package_name", "backup_storage_id", "backup_storage_access_info") {
return
}
pb := storageman.SStorageUnpackInstanceBackup{}
err := body.Unmarshal(&pb)
if err != nil {
hostutils.Response(ctx, w, httperrors.NewInputParameterError(err.Error()))
return
}
hostutils.DelayTask(ctx, unpackInstanceBackup, &pb)
hostutils.ResponseOk(ctx, w)
}
func packInstanceBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
sbParams := params.(*storageman.SStoragePackInstanceBackup)
backupStorage, err := backupstorage.GetBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
if err != nil {
return nil, err
}
err = backupStorage.InstancePack(sbParams.PackageName, sbParams.BackupIds, &sbParams.Metadata)
return nil, err
}
func unpackInstanceBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
sbParams := params.(*storageman.SStorageUnpackInstanceBackup)
backupStorage, err := backupstorage.GetBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
if err != nil {
return nil, err
}
diskBackupIds, metadata, err := backupStorage.InstanceUnpack(sbParams.PackageName)
if err != nil {
return nil, err
}
ret := jsonutils.NewDict()
ret.Set("disk_backup_ids", jsonutils.Marshal(diskBackupIds))
ret.Set("metadata", jsonutils.Marshal(metadata))
return ret, err
}
func storageDeleteBackup(ctx context.Context, w http.ResponseWriter, r *http.Request) {
_, _, body := appsrv.FetchEnv(ctx, w, r)
backupId, err := body.GetString("backup_id")
@@ -259,9 +323,20 @@ func storageDeleteBackup(ctx context.Context, w http.ResponseWriter, r *http.Req
hostutils.ResponseOk(ctx, w)
}
func checkOptions(ctx context.Context, w http.ResponseWriter, body jsonutils.JSONObject, options ...string) bool {
for _, option := range options {
if body.Contains(option) {
continue
}
hostutils.Response(ctx, w, httperrors.NewMissingParameterError(option))
return false
}
return true
}
func deleteBackup(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
sbParams := params.(*storageman.SStorageBackup)
backupStorage, err := backupstorage.NewBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
backupStorage, err := backupstorage.GetBackupStorage(sbParams.BackupStorageId, sbParams.BackupStorageAccessInfo)
if err != nil {
return nil, err
}
+22
View File
@@ -56,3 +56,25 @@ type SStorageBackup struct {
BackupStorageId string
BackupStorageAccessInfo *jsonutils.JSONDict
}
type SStoragePackBackup struct {
PackageName string
BackupId string
BackupStorageId string
BackupStorageAccessInfo *jsonutils.JSONDict
Metadata api.DiskBackupPackMetadata
}
type SStoragePackInstanceBackup struct {
PackageName string
BackupStorageId string
BackupStorageAccessInfo *jsonutils.JSONDict
BackupIds []string
Metadata api.InstanceBackupPackMetadata
}
type SStorageUnpackInstanceBackup struct {
PackageName string
BackupStorageId string
BackupStorageAccessInfo *jsonutils.JSONDict
}
+19
View File
@@ -144,3 +144,22 @@ func (opts *InstanceBackupRecoveryOptions) GetId() string {
func (opts *InstanceBackupRecoveryOptions) Params() (jsonutils.JSONObject, error) {
return jsonutils.Marshal(opts), nil
}
type InstanceBackupPackOptions struct {
DiskBackupIdOptions
PackageName string `help:"package name" json:"package_name"`
}
func (opts *InstanceBackupPackOptions) Params() (jsonutils.JSONObject, error) {
return jsonutils.Marshal(opts), nil
}
type InstanceBackupManagerCreateFromPackageOptions struct {
PackageName string `help:"package name" json:"package_name"`
Name string `help:"instance backup name" json:"name"`
BackupStorageId string `help:"backup storage id" json:"backup_storage_id"`
}
func (opts *InstanceBackupManagerCreateFromPackageOptions) Params() (jsonutils.JSONObject, error) {
return jsonutils.Marshal(opts), nil
}
+2
View File
@@ -216,6 +216,8 @@ const (
ACT_RESTART_NETWORK = "restart_network"
ACT_RECOVERY = "recovery"
ACT_PACK = "pack"
ACT_UNPACK = "unpack"
ACT_SYNC_CLASS_METADATA = "sync_class_metadata"
)