diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index e81216c734..43a4a7c3bb 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -3520,14 +3520,48 @@ func (self *SGuest) StartBlockIoThrottleTask(ctx context.Context, userCred mccli return nil } -func (self *SGuestManager) AllowPerformBatchMigrate(ctx context.Context, +func (manager *SGuestManager) AllowPerformBatchMigrate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool { - return db.IsAdminAllowPerform(userCred, self, "batch-guest-migrate") + return db.IsAdminAllowPerform(userCred, manager, "batch-guest-migrate") } -func (self *SGuestManager) PerformBatchMigrate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SGuest) validateForBatchMigrate(ctx context.Context) (*SGuest, error) { + guest := GuestManager.FetchGuestById(self.Id) + if guest.Hypervisor != api.HYPERVISOR_KVM { + return guest, httperrors.NewBadRequestError("guest %s hypervisor %s can't migrate", + guest.Name, guest.Hypervisor) + } + if len(guest.BackupHostId) > 0 { + return guest, httperrors.NewBadRequestError("guest %s has backup, can't migrate", guest.Name) + } + if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY, api.VM_UNKNOWN}) { + return guest, httperrors.NewBadRequestError("guest %s status %s can't migrate", guest.Name, guest.Status) + } + if guest.Status == api.VM_RUNNING { + if len(guest.GetIsolatedDevices()) > 0 { + return guest, httperrors.NewBadRequestError( + "guest %s status %s has isolated device, can't do migrate", + guest.Name, guest.Status, + ) + } + cdrom := guest.getCdrom(false) + if cdrom != nil && len(cdrom.ImageId) > 0 { + return guest, httperrors.NewBadRequestError("cannot migrate with cdrom") + } + } else if guest.Status == api.VM_UNKNOWN { + if guest.getDefaultStorageType() == api.STORAGE_LOCAL { + return guest, httperrors.NewBadRequestError( + "guest %s status %s can't migrate with local storage", + guest.Name, guest.Status, + ) + } + } + return guest, nil +} + +func (manager *SGuestManager) PerformBatchMigrate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { params := new(api.GuestBatchMigrateRequest) err := data.Unmarshal(params) if err != nil { @@ -3539,7 +3573,7 @@ func (self *SGuestManager) PerformBatchMigrate(ctx context.Context, userCred mcc var preferHostId string if len(params.PreferHost) > 0 { - if !db.IsAdminAllowPerform(userCred, self, "assign-host") { + if !db.IsAdminAllowPerform(userCred, manager, "assign-host") { return nil, httperrors.NewBadRequestError("Only system admin can assign host") } iHost, _ := HostManager.FetchByIdOrName(userCred, params.PreferHost) @@ -3560,35 +3594,13 @@ func (self *SGuestManager) PerformBatchMigrate(ctx context.Context, userCred mcc return nil, httperrors.NewBadRequestError("Check input guests is exist") } for i := 0; i < len(guests); i++ { - if guests[i].Hypervisor != api.HYPERVISOR_KVM { - return nil, httperrors.NewBadRequestError("guest %s hypervisor %s can't migrate", - guests[i].Name, guests[i].Hypervisor) - } - if len(guests[i].BackupHostId) > 0 { - return nil, httperrors.NewBadRequestError("guest %s has backup, can't migrate", guests[i].Name) - } - if !utils.IsInStringArray(guests[i].Status, []string{api.VM_RUNNING, api.VM_READY, api.VM_UNKNOWN}) { - return nil, httperrors.NewBadRequestError("guest %s status %s can't migrate", guests[i].Name, guests[i].Status) - } - if guests[i].Status == api.VM_RUNNING { - if len(guests[i].GetIsolatedDevices()) > 0 { - return nil, httperrors.NewBadRequestError( - "guest %s status %s has isolated device, can't do migrate", - guests[i].Name, guests[i].Status, - ) - } - cdrom := guests[i].getCdrom(false) - if cdrom != nil && len(cdrom.ImageId) > 0 { - return nil, httperrors.NewBadRequestError("cannot migrate with cdrom") - } - } else if guests[i].Status == api.VM_UNKNOWN { - if guests[i].getDefaultStorageType() == api.STORAGE_LOCAL { - return nil, httperrors.NewBadRequestError( - "guest %s status %s can't migrate with local storage", - guests[i].Name, guests[i].Status, - ) - } + lockman.LockObject(ctx, &guests[i]) + defer lockman.ReleaseObject(ctx, &guests[i]) + guest, err := guests[i].validateForBatchMigrate(ctx) + if err != nil { + return nil, err } + guests[i] = *guest } var hostGuests = map[string][]*api.GuestBatchMigrateParams{} @@ -3613,12 +3625,20 @@ func (self *SGuestManager) PerformBatchMigrate(ctx context.Context, userCred mcc kwargs.Set("prefer_host_id", jsonutils.NewString(preferHostId)) } host := HostManager.FetchHostById(hostId) - task, err := taskman.TaskManager.NewTask(ctx, "HostGuestsMigrateTask", host, userCred, kwargs, "", "", nil) - if err != nil { - log.Errorln(err) - continue - } - task.ScheduleRun(nil) + manager.StartHostGuestsMigrateTask(ctx, userCred, host, kwargs, "") } return nil, nil } + +func (manager *SGuestManager) StartHostGuestsMigrateTask( + ctx context.Context, userCred mcclient.TokenCredential, + host *SHost, kwargs *jsonutils.JSONDict, parentTaskId string, +) error { + task, err := taskman.TaskManager.NewTask(ctx, "HostGuestsMigrateTask", host, userCred, kwargs, parentTaskId, "", nil) + if err != nil { + log.Errorln(err) + return err + } + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index bf4f3ffa1b..41031bc5a3 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -4050,37 +4050,33 @@ func (host *SHost) PerformHostMaintenance(ctx context.Context, userCred mcclient guests := host.GetGuests() for i := 0; i < len(guests); i++ { - if !utils.IsInStringArray(guests[i].Status, []string{api.VM_READY, api.VM_RUNNING, api.VM_UNKNOWN}) { - return nil, httperrors.NewBadRequestError( - "guest %s(%s) status %s can't do migrate", - guests[i].Name, guests[i].Id, guests[i].Status) + lockman.LockObject(ctx, &guests[i]) + defer lockman.ReleaseObject(ctx, &guests[i]) + guest, err := guests[i].validateForBatchMigrate(ctx) + if err != nil { + return nil, err } - if len(guests[i].BackupHostId) > 0 { - return nil, httperrors.NewBadRequestError("Guest %s(%s) has backup guest", guests[i].Name, guests[i].Id) - } - if guests[i].Status == api.VM_RUNNING { - if len(guests[i].GetIsolatedDevices()) > 0 { - return nil, httperrors.NewBadRequestError( - "guest %s(%s) attach isolated device, can't migrate", - guests[i].Name, guests[i].Id) - } - cdrom := guests[i].getCdrom(false) - if cdrom != nil && len(cdrom.ImageId) > 0 { - return nil, httperrors.NewBadRequestError( - "Cannot migrate %s(%s) with cdrom", guests[i].Name, guests[i].Id) - } - } - if guests[i].Status == api.VM_UNKNOWN { - if guests[i].getDefaultStorageType() == api.STORAGE_LOCAL { - return nil, httperrors.NewBadRequestError( - "Cannot migrate guest %s(%s) status %s with local storage", - guests[i].Name, guests[i].Id, guests[i].Status) - } + guests[i] = *guest + if host.HostStatus == api.HOST_OFFLINE && guests[i].Status != api.VM_UNKNOWN { + return nil, httperrors.NewBadRequestError("Host %s can't migrate guests %s in status %s", + host.HostStatus, guests[i].Name, guests[i].Status) } + } + var hostGuests = []*api.GuestBatchMigrateParams{} + for i := 0; i < len(guests); i++ { + bmp := &api.GuestBatchMigrateParams{ + Id: guests[i].Id, + LiveMigrate: guests[i].Status == api.VM_RUNNING, + RescueMode: guests[i].Status == api.VM_UNKNOWN, + OldStatus: guests[i].Status, + } + guests[i].SetStatus(userCred, api.VM_START_MIGRATE, "host maintainence") + hostGuests = append(hostGuests, bmp) } kwargs := jsonutils.NewDict() + kwargs.Set("guests", jsonutils.Marshal(hostGuests)) kwargs.Set("prefer_host_id", jsonutils.NewString(preferHostId)) return nil, host.StartMaintainTask(ctx, userCred, kwargs) } diff --git a/pkg/compute/tasks/host_guests_migrate_task.go b/pkg/compute/tasks/host_guests_migrate_task.go index 8c07acd970..e7d0ac7aaa 100644 --- a/pkg/compute/tasks/host_guests_migrate_task.go +++ b/pkg/compute/tasks/host_guests_migrate_task.go @@ -58,7 +58,11 @@ func (self *HostGuestsMigrateTask) OnInit(ctx context.Context, obj db.IStandalon } } if !guestMigrating { - self.SetStageComplete(ctx, nil) + if jsonutils.QueryBoolean(self.Params, "some_guest_migrate_failed", false) { + self.SetStageFailed(ctx, "some guest migrate failed") + } else { + self.SetStageComplete(ctx, nil) + } } else { guests := append(guests[:migrateIndex], guests[migrateIndex+1:]...) params := jsonutils.NewDict() @@ -68,6 +72,8 @@ func (self *HostGuestsMigrateTask) OnInit(ctx context.Context, obj db.IStandalon } func (self *HostGuestsMigrateTask) OnInitFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - log.Errorf("HostGuestsMigrateTask on failed %s", data) + kwargs := jsonutils.NewDict() + kwargs.Set("some_guest_migrate_failed", jsonutils.JSONTrue) + self.SaveParams(kwargs) self.OnInit(ctx, obj, data) } diff --git a/pkg/compute/tasks/host_maintenance_task.go b/pkg/compute/tasks/host_maintenance_task.go index cc2e8cd1ba..993cdde4db 100644 --- a/pkg/compute/tasks/host_maintenance_task.go +++ b/pkg/compute/tasks/host_maintenance_task.go @@ -4,7 +4,6 @@ import ( "context" "yunion.io/x/jsonutils" - "yunion.io/x/log" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -23,74 +22,34 @@ func init() { func (self *HostMaintainTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { host := obj.(*models.SHost) - guests := host.GetGuests() - preferHostId, _ := data.GetString("prefer_host_id") + guests, _ := self.Params.Get("guests") + preferHostId, _ := self.Params.Get("prefer_host_id") - var doMigrate bool - var hasGuestCannotMigrate bool - for i := 0; i < len(guests); i++ { - if guests[i].Status == api.VM_READY || guests[i].Status == api.VM_UNKNOWN { - rescueMode := guests[i].Status == api.VM_UNKNOWN - if rescueMode { - guestDisks := guests[i].GetDisks() - var allDiskIsShared = true - for _, guestDisk := range guestDisks { - if guestDisk.GetDisk().GetStorage().StorageType == api.STORAGE_LOCAL { - allDiskIsShared = false - break - } - } - if !allDiskIsShared { - hasGuestCannotMigrate = true - continue - } - } - err := guests[i].StartMigrateTask(ctx, self.UserCred, rescueMode, false, guests[i].Status, preferHostId, self.Id) - if err != nil { - log.Errorf("Start migrate task failed %s", err) - hasGuestCannotMigrate = true - continue - } else { - log.Infof("Start migrate %s(%s)", guests[i].Name, guests[i].Id) - doMigrate = true - break - } - } else if guests[i].Status == api.VM_RUNNING { - err := guests[i].StartGuestLiveMigrateTask(ctx, self.UserCred, guests[i].Status, preferHostId, self.Id) - if err != nil { - log.Errorf("Start migrate task failed %s", err) - hasGuestCannotMigrate = true - continue - } else { - log.Infof("Start migrate %s(%s)", guests[i].Name, guests[i].Id) - doMigrate = true - break - } - } - } - if hasGuestCannotMigrate { - kwargs := jsonutils.NewDict() - kwargs.Set("some_guest_migrate_failed", jsonutils.JSONTrue) - self.SaveParams(kwargs) - } - - if !doMigrate { - hostStatus := api.HOST_MAINTAINING - if jsonutils.QueryBoolean(self.Params, "some_guest_migrate_failed", false) { - hostStatus = api.HOST_MAINTAIN_FAILE - } - host.PerformDisable(ctx, self.UserCred, nil, nil) - host.SetStatus(self.UserCred, hostStatus, "On host maintain task complete") - logclient.AddSimpleActionLog(host, logclient.ACT_HOST_MAINTAINING, "host maintain", self.UserCred, hostStatus == api.HOST_MAINTAINING) - self.SetStageComplete(ctx, nil) + kwargs := jsonutils.NewDict() + kwargs.Set("guests", guests) + kwargs.Set("prefer_host_id", preferHostId) + self.SetStage("OnGuestsMigrate", nil) + err := models.GuestManager.StartHostGuestsMigrateTask(ctx, self.UserCred, host, self.Params, self.Id) + if err != nil { + self.TaskFailed(ctx, host, err.Error()) return } } -// Ignore guest migrate fail -func (self *HostMaintainTask) OnInitFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { - kwargs := jsonutils.NewDict() - kwargs.Set("some_guest_migrate_failed", jsonutils.JSONTrue) - self.SaveParams(kwargs) - self.OnInit(ctx, obj, data) +func (self *HostMaintainTask) OnGuestsMigrate(ctx context.Context, host *models.SHost, data jsonutils.JSONObject) { + host.PerformDisable(ctx, self.UserCred, nil, nil) + host.SetStatus(self.UserCred, api.HOST_MAINTAINING, "On host maintain task complete") + logclient.AddSimpleActionLog(host, logclient.ACT_HOST_MAINTAINING, "host maintain", self.UserCred, true) + self.SetStageComplete(ctx, nil) +} + +func (self *HostMaintainTask) OnGuestsMigrateFailed(ctx context.Context, host *models.SHost, data jsonutils.JSONObject) { + self.TaskFailed(ctx, host, data.String()) +} + +func (self *HostMaintainTask) TaskFailed(ctx context.Context, host *models.SHost, reason string) { + host.PerformDisable(ctx, self.UserCred, nil, nil) + host.SetStatus(self.UserCred, api.HOST_MAINTAIN_FAILE, "On host maintain task complete failed") + logclient.AddSimpleActionLog(host, logclient.ACT_HOST_MAINTAINING, "host maintain", self.UserCred, false) + self.SetStageFailed(ctx, reason) }