diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index a5f8164269..5b5617b28c 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -3555,3 +3555,34 @@ func (self *SHost) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { desc.Update(jsonutils.Marshal(&info)) return desc } + +func (self *SHost) MarkGuestUnknown(userCred mcclient.TokenCredential) { + log.Errorln(self.GetGuests()) + for _, guest := range self.GetGuests() { + guest.SetStatus(userCred, VM_UNKNOWN, "host offline") + } +} + +func (manager *SHostManager) PingDetectionTask(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { + deadline := time.Now().Add(-1 * time.Duration(options.Options.HostOfflineMaxSeconds) * time.Second) + + q := manager.Query().Equals("host_status", HOST_ONLINE). + Equals("host_type", HOST_TYPE_HYPERVISOR) + q = q.Filter(sqlchemy.OR(sqlchemy.IsNull(q.Field("last_ping_at")), + sqlchemy.LT(q.Field("last_ping_at"), deadline))) + + rows, err := q.Rows() + if err != nil { + log.Errorln(err) + return + } + defer rows.Close() + + for rows.Next() { + var host = SHost{} + q.Row2Struct(rows, &host) + host.SetModelManager(manager) + host.PerformOffline(ctx, userCred, nil, nil) + host.MarkGuestUnknown(userCred) + } +} diff --git a/pkg/compute/options/options.go b/pkg/compute/options/options.go index 3e369f6527..cf2e426f91 100644 --- a/pkg/compute/options/options.go +++ b/pkg/compute/options/options.go @@ -76,6 +76,9 @@ type ComputeOptions struct { SnapshotCreateDiskProtocol string `help:"Snapshot create disk protocol" choices:"url|fuse" default:"fuse"` + HostOfflineMaxSeconds int `help:"Maximal seconds interval that a host considered offline during which it did not ping region, default is 3 minues" default:"180"` + HostOfflineDetectionInterval int `help:"Interval to check offline hosts, defualt is half a minute" default:"30"` + cloudcommon.DBOptions } diff --git a/pkg/compute/service/service.go b/pkg/compute/service/service.go index 398033c794..e803555a6e 100644 --- a/pkg/compute/service/service.go +++ b/pkg/compute/service/service.go @@ -3,6 +3,7 @@ package service import ( "os" "time" + "yunion.io/x/onecloud/pkg/compute/skus" "yunion.io/x/log" @@ -60,6 +61,7 @@ func StartService() { cron.AddJob1("CleanPendingDeleteDisks", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.DiskManager.CleanPendingDeleteDisks) cron.AddJob1("CleanPendingDeleteLoadbalancers", time.Duration(options.Options.LoadbalancerPendingDeleteCheckInterval)*time.Second, models.LoadbalancerAgentManager.CleanPendingDeleteLoadbalancers) cron.AddJob1("CleanPendingDeleteServers", time.Duration(options.Options.PrepaidExpireCheckSeconds)*time.Second, models.GuestManager.DeleteExpiredPrepaidServers) + cron.AddJob1("StartHostPingDetectionTask", time.Duration(options.Options.HostOfflineDetectionInterval)*time.Second, models.HostManager.PingDetectionTask) cron.AddJob2("AutoDiskSnapshot", options.Options.AutoSnapshotDay, options.Options.AutoSnapshotHour, 0, 0, models.DiskManager.AutoDiskSnapshot, false) cron.AddJob2("SyncSkus", options.Options.SyncSkusDay, options.Options.SyncSkusHour, 0, 0, skus.SyncSkus, true) diff --git a/pkg/compute/tasks/baremetal_sync_status_task.go b/pkg/compute/tasks/baremetal_sync_status_task.go index e94303a4ea..614572e78c 100644 --- a/pkg/compute/tasks/baremetal_sync_status_task.go +++ b/pkg/compute/tasks/baremetal_sync_status_task.go @@ -5,6 +5,7 @@ import ( "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" @@ -76,14 +77,17 @@ func (self *BaremetalSyncAllGuestsStatusTask) OnInit(ctx context.Context, obj db } func (self *BaremetalSyncAllGuestsStatusTask) OnGuestSyncStatusComplete(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { - guests := baremetal.GetGuests() - for _, guest := range guests { - if guest.Status == models.VM_UNKNOWN && guest.Hypervisor == models.HYPERVISOR_BAREMETAL { + var guests = make([]models.SGuest, 0) + for _, guest := range baremetal.GetGuests() { + if guest.Status == models.VM_UNKNOWN && guest.Hypervisor != models.HYPERVISOR_BAREMETAL { guest.SetStatus(self.UserCred, models.VM_SYNCING_STATUS, "") - // GuestBatchSyncStatusTask - guest.StartSyncstatus(ctx, self.UserCred, "") + guests = append(guests, guest) } } + for _, guest := range guests { + guest.StartSyncstatus(ctx, self.UserCred, "") + } + log.Infof("All unknown guests syncstatus complete") self.SetStageComplete(ctx, nil) }