From 918a9c43bd383c92fc8137a3536ff81d3deec164 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Sat, 21 Sep 2019 18:36:44 +0800 Subject: [PATCH] fix rbd disk delete rebuild root with snapshot --- pkg/compute/hostdrivers/kvm.go | 4 ++ pkg/compute/models/disks.go | 19 ++++++++ pkg/compute/models/snapshots.go | 38 ++++++++++++++- pkg/compute/models/storages.go | 11 +++++ pkg/compute/storagedrivers/rbd.go | 8 +--- pkg/compute/tasks/disk_create_task.go | 16 +++++++ pkg/compute/tasks/disk_delete_task.go | 47 +++++++++++++++++++ pkg/compute/tasks/disk_reset_task.go | 2 +- pkg/compute/tasks/guest_detach_disk_task.go | 10 +--- pkg/hostman/guestman/guestman.go | 8 +++- .../hostdeployer/deployserver/deployserver.go | 20 ++++---- pkg/hostman/storageman/disk_base.go | 5 ++ pkg/hostman/storageman/disk_local.go | 5 ++ pkg/hostman/storageman/disk_rbd.go | 13 +++++ pkg/hostman/storageman/storage_base.go | 2 +- 15 files changed, 176 insertions(+), 32 deletions(-) diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index 9fba2f99b5..961f69fd71 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -272,6 +272,10 @@ func (self *SKVMHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, ho func (self *SKVMHostDriver) RequestRebuildDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error { content.Add(jsonutils.JSONTrue, "rebuild") + if task.GetParams().Contains("backing_disk_id") { + backingDiskId, _ := task.GetParams().GetString("backing_disk_id") + content.Set("backing_disk_id", jsonutils.NewString(backingDiskId)) + } return self.RequestAllocateDiskOnStorage(ctx, host, storage, disk, task, content) } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index 6eadece4e6..15cd1d1c1d 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -2098,3 +2098,22 @@ func (self *SDisk) IsNeedWaitSnapshotsDeleted() (bool, error) { } return false, nil } + +func (self *SDisk) UpdataSnapshotsBackingDisk(backingDiskId string) error { + snapshots := make([]SSnapshot, 0) + err := SnapshotManager.Query().Equals("disk_id", self.Id).IsNullOrEmpty("backing_disk_id").All(&snapshots) + if err != nil { + return err + } + for i := 0; i < len(snapshots); i++ { + snapshots[i].SetModelManager(SnapshotManager, &snapshots[i]) + _, err := db.Update(&snapshots[i], func() error { + snapshots[i].BackingDiskId = backingDiskId + return nil + }) + if err != nil { + log.Errorln(err) + } + } + return nil +} diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index 2492156885..749544c166 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/compare" "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -64,6 +65,7 @@ type SSnapshot struct { RefCount int `nullable:"false" default:"0" list:"user"` CloudregionId string `width:"36" charset:"ascii" nullable:"true" list:"user" create:"optional"` + BackingDiskId string `width:"36" charset:"ascii" nullable:"true" default:""` } var SnapshotManager *SSnapshotManager @@ -555,8 +557,9 @@ func (self *SSnapshot) StartSnapshotsDeleteTask(ctx context.Context, userCred mc func (self *SSnapshot) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { if len(self.DiskId) > 0 { + storage := self.GetStorage() disk := DiskManager.FetchDiskById(self.DiskId) - if disk != nil && disk.GetStorage().StorageType == api.STORAGE_RBD { + if disk != nil && storage.StorageType == api.STORAGE_RBD { cnt, err := disk.GetGuestsCount() if err == nil { val := disk.GetMetadata("disk_delete_after_snapshots", userCred) @@ -566,11 +569,44 @@ func (self *SSnapshot) RealDelete(ctx context.Context, userCred mcclient.TokenCr } else { log.Errorln(err) } + } else { + if storage.StorageType == api.STORAGE_RBD { + backingDisks, err := self.GetBackingDisks() + if err != nil { + log.Errorln(err) + } else { + storage.StartDeleteRbdDisks(ctx, userCred, backingDisks) + } + } } } return db.DeleteModel(ctx, userCred, self) } +func (self *SSnapshot) GetBackingDisks() ([]string, error) { + count, err := SnapshotManager.Query().Equals("disk_id", self.DiskId).IsNullOrEmpty("backing_disk_id").CountWithError() + if err != nil { + return nil, err + } + if count > 0 { + return nil, nil + } else { + sps := make([]SSnapshot, 0) + err := SnapshotManager.Query().Equals("disk_id", self.DiskId).All(&sps) + if err != nil { + return nil, err + } + res := make([]string, 0) + for i := 0; i < len(sps); i++ { + if len(sps[i].BackingDiskId) > 0 && !utils.IsInStringArray(sps[i].BackingDiskId, res) { + res = append(res, sps[i].BackingDiskId) + } + } + res = append(res, self.DiskId) + return res, nil + } +} + func (self *SSnapshot) FakeDelete(userCred mcclient.TokenCredential) error { _, err := db.Update(self, func() error { self.FakeDeleted = true diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 304fbf48a8..f1bb0f6379 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -1274,3 +1274,14 @@ func (manager *SStorageManager) StorageSnapshotsRecycle(ctx context.Context, use } } } + +func (self *SStorage) StartDeleteRbdDisks(ctx context.Context, userCred mcclient.TokenCredential, disksId []string) error { + data := jsonutils.NewDict() + data.Add(jsonutils.NewStringArray(disksId), "disks_id") + task, err := taskman.TaskManager.NewTask(ctx, "StorageDeleteRbdDiskTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/storagedrivers/rbd.go b/pkg/compute/storagedrivers/rbd.go index f4c7eedc8f..8bbcd03c99 100644 --- a/pkg/compute/storagedrivers/rbd.go +++ b/pkg/compute/storagedrivers/rbd.go @@ -201,17 +201,13 @@ func (self *SRbdStorageDriver) RequestCreateSnapshot(ctx context.Context, snapsh } func (self *SRbdStorageDriver) RequestDeleteSnapshot(ctx context.Context, snapshot *models.SSnapshot, task taskman.ITask) error { - disk, err := snapshot.GetDisk() - if err != nil { - return errors.Wrap(err, "snapshot get disk") - } storage := snapshot.GetStorage() host := storage.GetMasterHost() - url := fmt.Sprintf("%s/disks/%s/delete-snapshot/%s", host.ManagerUri, storage.Id, disk.Id) + url := fmt.Sprintf("%s/disks/%s/delete-snapshot/%s", host.ManagerUri, storage.Id, snapshot.DiskId) header := task.GetTaskRequestHeader() params := jsonutils.NewDict() params.Set("snapshot_id", jsonutils.NewString(snapshot.Id)) - _, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, params, false) + _, _, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, params, false) if err != nil { return errors.Wrap(err, "request create snapshot") } diff --git a/pkg/compute/tasks/disk_create_task.go b/pkg/compute/tasks/disk_create_task.go index fd7cdf4a5b..9d29e99fc6 100644 --- a/pkg/compute/tasks/disk_create_task.go +++ b/pkg/compute/tasks/disk_create_task.go @@ -20,6 +20,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/util/stringutils" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -69,6 +70,12 @@ func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, dis if len(disk.BackupStorageId) > 0 { self.SetStage("OnMasterStorageCreateDiskComplete", nil) } else { + if rebuild && storage.StorageType == api.STORAGE_RBD { + if count, _ := disk.GetSnapshotCount(); count > 0 { + backingDiskId := stringutils.UUID4() + self.Params.Set("backing_disk_id", jsonutils.NewString(backingDiskId)) + } + } self.SetStage("OnDiskReady", nil) } if err := disk.StartAllocate(ctx, host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { @@ -109,6 +116,15 @@ func (self *DiskCreateTask) OnDiskReady(ctx context.Context, disk *models.SDisk, }); err != nil { log.Errorf("update disk info error: %v", err) } + if jsonutils.QueryBoolean(self.Params, "rebuild", false) { + backingDiskId, _ := self.Params.GetString("backing_disk_id") + if len(backingDiskId) > 0 { + err := disk.UpdataSnapshotsBackingDisk(backingDiskId) + if err != nil { + log.Errorf("update disk snapshots backing disk fiailed %s", err) + } + } + } disk.SetStatus(self.UserCred, api.DISK_READY, "") self.CleanHostSchedCache(disk) diff --git a/pkg/compute/tasks/disk_delete_task.go b/pkg/compute/tasks/disk_delete_task.go index 0249e66960..4a7d23dedf 100644 --- a/pkg/compute/tasks/disk_delete_task.go +++ b/pkg/compute/tasks/disk_delete_task.go @@ -19,6 +19,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -33,6 +34,7 @@ type DiskDeleteTask struct { func init() { taskman.RegisterTask(DiskDeleteTask{}) + taskman.RegisterTask(StorageDeleteRbdDiskTask{}) } func (self *DiskDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { @@ -89,6 +91,10 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD if isPurge { self.OnGuestDiskDeleteComplete(ctx, disk, nil) } else { + if isNeed, _ := disk.IsNeedWaitSnapshotsDeleted(); isNeed { + self.OnGuestDiskDeleteComplete(ctx, disk, nil) + return + } if len(disk.BackupStorageId) > 0 { self.SetStage("OnMasterStorageDeleteDiskComplete", nil) } else { @@ -153,3 +159,44 @@ func (self *DiskDeleteTask) OnGuestDiskDeleteCompleteFailed(ctx context.Context, self.SetStageFailed(ctx, reason.String()) db.OpsLog.LogEvent(disk, db.ACT_DELOCATE_FAIL, disk.GetShortDesc(ctx), self.GetUserCred()) } + +type StorageDeleteRbdDiskTask struct { + taskman.STask +} + +func (self *StorageDeleteRbdDiskTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + storage := obj.(*models.SStorage) + self.DeleteDisk(ctx, storage, self.Params) +} + +func (self *StorageDeleteRbdDiskTask) OnDeleteDisk(ctx context.Context, storage *models.SStorage, data jsonutils.JSONObject) { + self.DeleteDisk(ctx, storage, data) +} + +func (self *StorageDeleteRbdDiskTask) DeleteDisk(ctx context.Context, storage *models.SStorage, data jsonutils.JSONObject) { + disksId := make([]string, 0) + data.Unmarshal(&disksId, "disks_id") + if len(disksId) == 0 { + self.SetStageComplete(ctx, nil) + return + } + params := jsonutils.NewDict() + params.Set("disks_id", jsonutils.Marshal(disksId[1:])) + params.Set("delete_disk", jsonutils.NewString(disksId[0])) + self.SetStage("OnDeleteDisk", params) + header := self.GetTaskRequestHeader() + url := fmt.Sprintf("/disks/%s/delete/%s", storage.Id, disksId[0]) + body := jsonutils.NewDict() + host := storage.GetMasterHost() + _, err := host.Request(ctx, self.GetUserCred(), "POST", url, header, body) + if err != nil { + log.Errorln(err) + self.OnDeleteDiskFailed(ctx, storage, params) + } +} + +func (self *StorageDeleteRbdDiskTask) OnDeleteDiskFailed(ctx context.Context, storage *models.SStorage, data jsonutils.JSONObject) { + deleteDisk, _ := data.GetString("delete_disk") + db.OpsLog.LogEvent(storage, db.ACT_DELETE_OBJECT, fmt.Sprintf("delete disk %s failed", deleteDisk), self.UserCred) + self.DeleteDisk(ctx, storage, self.Params) +} diff --git a/pkg/compute/tasks/disk_reset_task.go b/pkg/compute/tasks/disk_reset_task.go index ae79d4fa22..4dd476c4f0 100644 --- a/pkg/compute/tasks/disk_reset_task.go +++ b/pkg/compute/tasks/disk_reset_task.go @@ -171,7 +171,7 @@ func (self *DiskCleanUpSnapshotsTask) OnCleanUpSnapshots(ctx context.Context, di } deleteSnapshots, _ := self.Params.GetArray("delete_snapshots") for i := 0; i < len(deleteSnapshots); i++ { - snapshot_id, _ := convertSnapshots[i].GetString() + snapshot_id, _ := deleteSnapshots[i].GetString() iSnapshot, err := models.SnapshotManager.FetchById(snapshot_id) if err != nil { log.Errorf("OnCleanUpSnapshots Fetch snapshot by id(%s) error:%s", snapshot_id, err.Error()) diff --git a/pkg/compute/tasks/guest_detach_disk_task.go b/pkg/compute/tasks/guest_detach_disk_task.go index 4f9c3bf0bb..459712d386 100644 --- a/pkg/compute/tasks/guest_detach_disk_task.go +++ b/pkg/compute/tasks/guest_detach_disk_task.go @@ -98,12 +98,7 @@ func (self *GuestDetachDiskTask) OnDetachDiskComplete(ctx context.Context, guest if host != nil && !host.Enabled && jsonutils.QueryBoolean(self.Params, "purge", false) { purge = true } - waitSnapshotsDelete, err := disk.IsNeedWaitSnapshotsDeleted() - if err != nil { - self.OnTaskFail(ctx, guest, disk, err) - return - } - if !keepDisk && disk.AutoDelete && !waitSnapshotsDelete { + if !keepDisk && disk.AutoDelete { cnt, _ := disk.GetGuestDiskCount() if cnt == 0 { self.SetStage("OnDiskDeleteComplete", nil) @@ -115,9 +110,6 @@ func (self *GuestDetachDiskTask) OnDetachDiskComplete(ctx context.Context, guest return } } - if waitSnapshotsDelete { - disk.SetMetadata(ctx, "disk_delete_after_snapshots", "true", self.UserCred) - } self.OnDiskDeleteComplete(ctx, guest, nil) } diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 50db11bed0..f7753a3dbf 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -351,13 +351,17 @@ func (m *SGuestManager) GuestCreate(ctx context.Context, params interface{}) (js guest = NewKVMGuestInstance(deployParams.Sid, m) desc, _ := deployParams.Body.Get("desc") if desc != nil { - err := guest.SaveDesc(desc) + err := guest.PrepareDir() + if err != nil { + return errors.Wrap(err, "guest prepare dir") + } + err = guest.SaveDesc(desc) if err != nil { return errors.Wrap(err, "save desc") } } m.SaveServer(deployParams.Sid, guest) - return guest.PrepareDir() + return nil }() if err != nil { return nil, errors.Wrap(err, "prepare guest") diff --git a/pkg/hostman/hostdeployer/deployserver/deployserver.go b/pkg/hostman/hostdeployer/deployserver/deployserver.go index 2f5eb3106a..1b36bfe06b 100644 --- a/pkg/hostman/hostdeployer/deployserver/deployserver.go +++ b/pkg/hostman/hostdeployer/deployserver/deployserver.go @@ -77,20 +77,16 @@ func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsPara } root := disk.MountKvmRootfs() - if root == nil { - // is not a root fs - goto resizePartition - } else if root.IsResizeFsPartitionSupport() { - goto resizePartition + if root != nil { + defer disk.UmountKvmRootfs(root) } - disk.UmountKvmRootfs(root) - return new(deployapi.Empty), nil - -resizePartition: - disk.UmountKvmRootfs(root) - err := disk.ResizePartition() - return new(deployapi.Empty), err + if root != nil && !root.IsResizeFsPartitionSupport() { + return new(deployapi.Empty), nil + } else { + err := disk.ResizePartition() + return new(deployapi.Empty), err + } } func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsParams) (*deployapi.Empty, error) { diff --git a/pkg/hostman/storageman/disk_base.go b/pkg/hostman/storageman/disk_base.go index 2254fa0ef0..95db9d84a5 100644 --- a/pkg/hostman/storageman/disk_base.go +++ b/pkg/hostman/storageman/disk_base.go @@ -36,6 +36,7 @@ type IDisk interface { GetDiskDesc() jsonutils.JSONObject GetDiskSetupScripts(idx int) string GetSnapshotLocation() string + OnRebuildRoot(ctx context.Context, params jsonutils.JSONObject) error DeleteAllSnapshot() error DiskSnapshot(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) @@ -89,6 +90,10 @@ func (d *SBaseDisk) Delete(ctx context.Context, params interface{}) (jsonutils.J return nil, fmt.Errorf("Not implemented") } +func (d *SBaseDisk) OnRebuildRoot(ctx context.Context, params jsonutils.JSONObject) error { + return fmt.Errorf("Not implemented") +} + func (d *SBaseDisk) CreateFromUrl(ctx context.Context, url string, size int64) error { return fmt.Errorf("Not implemented") } diff --git a/pkg/hostman/storageman/disk_local.go b/pkg/hostman/storageman/disk_local.go index 712f155314..0e3df75693 100644 --- a/pkg/hostman/storageman/disk_local.go +++ b/pkg/hostman/storageman/disk_local.go @@ -119,6 +119,11 @@ func (d *SLocalDisk) Delete(ctx context.Context, params interface{}) (jsonutils. return nil, nil } +func (d *SLocalDisk) OnRebuildRoot(ctx context.Context, params jsonutils.JSONObject) error { + _, err := d.Delete(ctx, params) + return err +} + func (d *SLocalDisk) Resize(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { diskInfo, ok := params.(*jsonutils.JSONDict) if !ok { diff --git a/pkg/hostman/storageman/disk_rbd.go b/pkg/hostman/storageman/disk_rbd.go index a8a249a9ca..d418f948d1 100644 --- a/pkg/hostman/storageman/disk_rbd.go +++ b/pkg/hostman/storageman/disk_rbd.go @@ -98,6 +98,19 @@ func (d *SRBDDisk) Delete(ctx context.Context, params interface{}) (jsonutils.JS return nil, storage.deleteImage(pool, d.Id) } +func (d *SRBDDisk) OnRebuildRoot(ctx context.Context, params jsonutils.JSONObject) error { + backingDiskId, err := params.GetString("backing_disk_id") + if err != nil { + _, err := d.Delete(ctx, params) + return err + } else { + storage := d.Storage.(*SRbdStorage) + storageConf := d.Storage.GetStorageConf() + pool, _ := storageConf.GetString("pool") + return storage.renameImage(pool, d.Id, backingDiskId) + } +} + func (d *SRBDDisk) Resize(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) { diskInfo, ok := params.(*jsonutils.JSONDict) if !ok { diff --git a/pkg/hostman/storageman/storage_base.go b/pkg/hostman/storageman/storage_base.go index f536f0308b..ecab32329a 100644 --- a/pkg/hostman/storageman/storage_base.go +++ b/pkg/hostman/storageman/storage_base.go @@ -243,7 +243,7 @@ func (s *SBaseStorage) CreateDiskByDiskinfo(ctx context.Context, params interfac if !jsonutils.QueryBoolean(createParams.DiskInfo, "rebuild", false) { return nil, fmt.Errorf("Disk exist") } - if _, err := createParams.Disk.Delete(ctx, params); err != nil { + if err := createParams.Disk.OnRebuildRoot(ctx, createParams.DiskInfo); err != nil { return nil, err } }