From baa7b9e09330352afd2e1088a9f1b837cb12bd09 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Mon, 13 Aug 2018 18:27:47 +0800 Subject: [PATCH] region: server-rebuid-root and fix bug on disk create and disk create failed --- pkg/cloudcommon/db/taskman/tasks.go | 6 +- pkg/compute/guestdrivers/base.go | 4 + pkg/compute/guestdrivers/kvm.go | 9 ++ pkg/compute/models/cachedimages.go | 6 +- pkg/compute/models/disks.go | 2 +- pkg/compute/models/guestdrivers.go | 1 + pkg/compute/models/guests.go | 59 ++++++++ pkg/compute/tasks/disk_create_task.go | 18 +-- pkg/compute/tasks/guest_rebuild_root_task.go | 140 +++++++++++++++++++ pkg/compute/tasks/guest_sync_task.go | 14 +- 10 files changed, 238 insertions(+), 21 deletions(-) create mode 100644 pkg/compute/tasks/guest_rebuild_root_task.go diff --git a/pkg/cloudcommon/db/taskman/tasks.go b/pkg/cloudcommon/db/taskman/tasks.go index ffeb1c1de0..bd8578ca36 100644 --- a/pkg/cloudcommon/db/taskman/tasks.go +++ b/pkg/cloudcommon/db/taskman/tasks.go @@ -307,7 +307,7 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, var stageName string if taskFailed { - stageName = fmt.Sprintf("%s_failed", task.Stage) + stageName = fmt.Sprintf("%sFailed", task.Stage) } else { stageName = task.Stage } @@ -315,6 +315,10 @@ func execITask(taskValue reflect.Value, task *STask, data jsonutils.JSONObject, funcValue := taskValue.MethodByName(stageName) if !funcValue.IsValid() || funcValue.IsNil() { + log.Debugf("Stage %s not found, try kebab to camel and find again", stageName) + if taskFailed { + stageName = fmt.Sprintf("%s_failed", task.Stage) + } stageName = utils.Kebab2Camel(stageName, "_") funcValue = taskValue.MethodByName(stageName) diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index fe123c5efd..b73a757637 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -117,3 +117,7 @@ func (self *SBaseGuestDriver) RequestChangeVmConfig(ctx context.Context, guest * func (self *SBaseGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { return fmt.Errorf("Not Implement") } + +func (self *SBaseGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return fmt.Errorf("Not Implement") +} diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 029e9bc80c..2a04d43eda 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -251,3 +251,12 @@ func (self *SKVMGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, gue func (self *SKVMGuestDriver) RequestGuestHotAddIso(ctx context.Context, guest *models.SGuest, path string, task taskman.ITask) error { return guest.StartSyncstatus(ctx, task.GetUserCred(), task.GetTaskId()) } + +func (self *SKVMGuestDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + subtask, err := taskman.TaskManager.NewTask(ctx, "KVMGuestRebuildRootTask", guest, task.GetUserCred(), nil, task.GetTaskId(), "", nil) + if err != nil { + return err + } + subtask.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go index a6be51853b..827fdc5d29 100644 --- a/pkg/compute/models/cachedimages.go +++ b/pkg/compute/models/cachedimages.go @@ -171,7 +171,7 @@ func (manager *SCachedimageManager) cacheImageInfo(ctx context.Context, userCred } } -func (manager *SCachedimageManager) getImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { +func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { if !refresh { imgObj, _ := manager.FetchById(imageId) if imgObj != nil { @@ -208,7 +208,7 @@ func (manager *SCachedimageManager) getImageByName(ctx context.Context, userCred } func (manager *SCachedimageManager) getImageInfo(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*SImage, error) { - img, err := manager.getImageById(ctx, userCred, imageId, refresh) + img, err := manager.GetImageById(ctx, userCred, imageId, refresh) if err == nil { return img, nil } @@ -237,7 +237,7 @@ func (self *SCachedimage) AllowPerformRefresh(ctx context.Context, userCred mccl } func (self *SCachedimage) PerformRefresh(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - img, err := CachedimageManager.getImageById(ctx, userCred, self.Id, true) + img, err := CachedimageManager.GetImageById(ctx, userCred, self.Id, true) if err != nil { return nil, err } diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index a47fdf3f0b..88608db85a 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -274,7 +274,7 @@ func (disk *SDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCreden func (self *SDisk) StartDiskCreateTask(ctx context.Context, userCred mcclient.TokenCredential, rebuild bool, snapshot string, parentTaskId string) error { kwargs := jsonutils.NewDict() if rebuild { - kwargs.Add(jsonutils.JSONFalse, "rebuild") + kwargs.Add(jsonutils.JSONTrue, "rebuild") } if len(snapshot) > 0 { kwargs.Add(jsonutils.NewString(snapshot), "snapshot") diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index ff978b3884..9e9ce6c70c 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -87,6 +87,7 @@ type IGuestDriver interface { RequestChangeVmConfig(ctx context.Context, guest *SGuest, task taskman.ITask, vcpuCount, vmemSize int64) error RequestGuestHotAddIso(ctx context.Context, guest *SGuest, path string, task taskman.ITask) error + RequestRebuildRootDisk(ctx context.Context, guest *SGuest, task taskman.ITask) error } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 81eb493186..80a177581b 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2145,6 +2145,65 @@ func (self *SGuest) PerformPurge(ctx context.Context, userCred mcclient.TokenCre return nil, err } +func (self *SGuest) AllowPerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { + return self.IsOwner(userCred) +} + +func (self *SGuest) PerformRebuildRoot(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + imageId, _ := data.GetString("image_id") + if utils.IsInStringArray(self.Status, []string{VM_READY, VM_RUNNING, VM_ADMIN}) { + if !data.Contains("image_id") { + gdc := self.CategorizeDisks() + imageId = gdc.Root.GetTemplateId() + if len(imageId) == 0 { + return nil, httperrors.NewBadRequestError("No template for root disk") + } + img, err := CachedimageManager.getImageInfo(ctx, userCred, imageId, false) + if err != nil { + return nil, httperrors.NewBadRequestError("Template %s not accessible: %s", imageId, err.Error()) + } + osType, _ := img.Properties["os_type"] + osName := self.GetMetadata("os_name", userCred) + if len(osName) == 0 && len(osType) == 0 && strings.ToLower(osType) != strings.ToLower(osName) { + return nil, httperrors.NewBadRequestError("Cannot switch OS between %s-%s", osName, osType) + } + } + autoStart := jsonutils.QueryBoolean(data, "auto_start", false) + var needStop = false + if self.Status == VM_RUNNING { + needStop = true + } + err := self.StartRebuildRootTask(ctx, userCred, imageId, needStop, autoStart) + return nil, err + } + return nil, httperrors.NewInvalidStatusError("Cannot reset root in status %s", self.Status) +} + +func (self *SGuest) StartRebuildRootTask(ctx context.Context, userCred mcclient.TokenCredential, imageId string, needStop, autoStart bool) error { + data := jsonutils.NewDict() + data.Set("image_id", jsonutils.NewString(imageId)) + if needStop { + data.Set("need_stop", jsonutils.JSONTrue) + } + if autoStart { + data.Set("auto_start", jsonutils.JSONTrue) + } + if self.GetHypervisor() == HYPERVISOR_BAREMETAL { + task, err := taskman.TaskManager.NewTask(ctx, "BaremetalServerRebuildRootTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + } else { + task, err := taskman.TaskManager.NewTask(ctx, "GuestRebuildRootTask", self, userCred, data, "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + } + return nil +} + func (self *SGuest) DetachDisk(ctx context.Context, disk *SDisk, userCred mcclient.TokenCredential) { guestdisk := self.GetGuestDisk(disk.Id) if guestdisk != nil { diff --git a/pkg/compute/tasks/disk_create_task.go b/pkg/compute/tasks/disk_create_task.go index c035d9e256..2466e9a441 100644 --- a/pkg/compute/tasks/disk_create_task.go +++ b/pkg/compute/tasks/disk_create_task.go @@ -27,11 +27,11 @@ func (self *DiskCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, self.SetStage("on_storage_cache_image_complete", nil) storagecache.StartImageCacheTask(ctx, self.UserCred, imageId, false, self.GetTaskId()) } else { - self.OnStorageCacheImageComplete(ctx, disk) + self.OnStorageCacheImageComplete(ctx, disk, nil) } } -func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, disk *models.SDisk) { +func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { rebuild, _ := self.GetParams().Bool("rebuild") snapshot, _ := self.GetParams().GetString("snapshot") if rebuild { @@ -43,13 +43,13 @@ func (self *DiskCreateTask) OnStorageCacheImageComplete(ctx context.Context, dis disk.SetStatus(self.GetUserCred(), models.DISK_STARTALLOC, "") self.SetStage("on_disk_ready", nil) if err := disk.StartAllocate(host, storage, self.GetTaskId(), self.GetUserCred(), rebuild, snapshot, self); err != nil { - self.OnStartAllocateFailed(ctx, disk, err) + self.OnStartAllocateFailed(ctx, disk, jsonutils.NewString(err.Error())) } } -func (self *DiskCreateTask) OnStartAllocateFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, resion.Error()) - self.SetStageFailed(ctx, resion.Error()) +func (self *DiskCreateTask) OnStartAllocateFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, data.String()) + self.SetStageFailed(ctx, data.String()) } func (self *DiskCreateTask) OnDiskReady(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { @@ -63,7 +63,7 @@ func (self *DiskCreateTask) OnDiskReady(ctx context.Context, disk *models.SDisk, self.SetStageComplete(ctx, nil) } -func (self *DiskCreateTask) OnDiskReadyFailed(ctx context.Context, disk *models.SDisk, resion error) { - disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, resion.Error()) - self.SetStageFailed(ctx, resion.Error()) +func (self *DiskCreateTask) OnDiskReadyFailed(ctx context.Context, disk *models.SDisk, data jsonutils.JSONObject) { + disk.SetStatus(self.UserCred, models.DISK_ALLOC_FAILED, data.String()) + self.SetStageFailed(ctx, data.String()) } diff --git a/pkg/compute/tasks/guest_rebuild_root_task.go b/pkg/compute/tasks/guest_rebuild_root_task.go new file mode 100644 index 0000000000..0d0ee7ee6a --- /dev/null +++ b/pkg/compute/tasks/guest_rebuild_root_task.go @@ -0,0 +1,140 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/osprofile" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" + "yunion.io/x/onecloud/pkg/compute/models" +) + +func init() { + taskman.RegisterTask(GuestRebuildRootTask{}) + taskman.RegisterTask(KVMGuestRebuildRootTask{}) +} + +type GuestRebuildRootTask struct { + SGuestBaseTask +} + +func (self *GuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if jsonutils.QueryBoolean(self.Params, "need_stop", false) { + self.SetStage("OnStopServerComplete", nil) + guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId()) + } else { + self.StartRebuildRootDisk(ctx, guest) + } +} + +func (self *GuestRebuildRootTask) StartRebuildRootDisk(ctx context.Context, guest *models.SGuest) { + db.OpsLog.LogEvent(guest, db.ACT_REBUILDING_ROOT, nil, self.UserCred) + gds := guest.CategorizeDisks() + imageId, _ := self.Params.GetString("image_id") + oldStatus := gds.Root.Status + _, err := gds.Root.GetModelManager().TableSpec().Update(gds.Root, func() error { + gds.Root.TemplateId = imageId + gds.Root.Status = models.DISK_REBUILD + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } else { + db.OpsLog.LogEvent(gds.Root, db.ACT_UPDATE_STATUS, + fmt.Sprintf("%s=>%s", oldStatus, models.DISK_REBUILD), self.UserCred) + } + + self.SetStage("OnRebuildRootDiskComplete", nil) + guest.SetStatus(self.UserCred, models.VM_REBUILD_ROOT, "") + guest.GetDriver().RequestRebuildRootDisk(ctx, guest, self) +} + +func (self *GuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + imgId, _ := self.Params.GetString("image_id") + imginfo, err := models.CachedimageManager.GetImageById(ctx, self.UserCred, imgId, false) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + osprof, err := osprofile.GetOSProfileFromImageProperties(imginfo.Properties, guest.Hypervisor) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + err = guest.SetMetadata(ctx, "__os_profile__", osprof, self.UserCred) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + if guest.OsType != osprof.OSType { + _, err := guest.GetModelManager().TableSpec().Update(guest, func() error { + guest.OsType = osprof.OSType + return nil + }) + if err != nil { + self.SetStageFailed(ctx, err.Error()) + return + } + } + db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT, "", self.UserCred) + // TODO: logclient + guest.NotifyServerEvent(notifyclient.SERVER_REBUILD_ROOT, notifyclient.PRIORITY_IMPORTANT, true) + self.SetStage("OnSyncStatusComplete", nil) + guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId()) +} + +func (self *GuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + db.OpsLog.LogEvent(guest, db.ACT_REBUILD_ROOT_FAIL, data.String(), self.UserCred) + guest.SetStatus(self.UserCred, models.VM_REBUILD_ROOT_FAIL, "") + // TODO: logclient +} + +func (self *GuestRebuildRootTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + if guest.Status == models.VM_READY && jsonutils.QueryBoolean(self.Params, "auto_start", false) { + self.SetStage("OnGuestStartComplete", nil) + guest.StartGueststartTask(ctx, self.UserCred, nil, self.GetTaskId()) + } else { + self.SetStageComplete(ctx, nil) + } +} + +func (self *GuestRebuildRootTask) OnGuestStartComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +/* -------------------------------------------------- */ +/* ------------ KVMGuestRebuildRootTask ------------- */ +/* -------------------------------------------------- */ + +type KVMGuestRebuildRootTask struct { + SGuestBaseTask +} + +func (self *KVMGuestRebuildRootTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + gds := guest.CategorizeDisks() + self.SetStage("OnRebuildRootDiskComplete", nil) + gds.Root.StartDiskCreateTask(ctx, self.UserCred, true, "", self.GetTaskId()) +} + +func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStage("OnGuestDeployComplete", nil) + guest.SetStatus(self.UserCred, models.VM_DEPLOYING, "") + params := jsonutils.NewDict() + params.Set("reset_password", jsonutils.JSONTrue) + guest.StartGuestDeployTask(ctx, self.UserCred, params, "deploy", self.GetTaskId()) +} + +func (self *KVMGuestRebuildRootTask) OnRebuildRootDiskCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageFailed(ctx, data.String()) +} + +func (self *KVMGuestRebuildRootTask) OnGuestDeployComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/compute/tasks/guest_sync_task.go b/pkg/compute/tasks/guest_sync_task.go index 5078a3a7ac..433fbfaa61 100644 --- a/pkg/compute/tasks/guest_sync_task.go +++ b/pkg/compute/tasks/guest_sync_task.go @@ -51,17 +51,17 @@ func (self *GuestSyncConfTask) OnDiskSyncComplete(ctx context.Context, guest *mo guest.StartSyncstatus(ctx, self.GetUserCred(), self.GetTaskId()) } -func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, resion error) { +func (self *GuestSyncConfTask) OnDiskSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) - db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, resion.Error(), self.UserCred) - log.Errorf("Guest sync config failed: %v", resion) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, data.String(), self.UserCred) + log.Errorf("Guest sync config failed: %v", data.String()) } -func (self *GuestSyncConfTask) OnSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, resion error) { +func (self *GuestSyncConfTask) OnSyncCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { guest := obj.(*models.SGuest) - guest.SetStatus(self.GetUserCred(), models.VM_SYNC_FAIL, resion.Error()) - log.Errorf("Guest sync config failed: %v", resion) - db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, resion.Error(), self.UserCred) + guest.SetStatus(self.GetUserCred(), models.VM_SYNC_FAIL, data.String()) + log.Errorf("Guest sync config failed: %v", data.String()) + db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF_FAIL, data.String(), self.UserCred) } func (self *GuestSyncConfTask) OnSyncStatusComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {