20181204-20181205

This commit is contained in:
wanyaoqi
2018-12-05 13:27:15 +08:00
parent e25730acce
commit 837502a068
12 changed files with 331 additions and 97 deletions
+4
View File
@@ -1,4 +1,8 @@
package main
import "yunion.io/x/onecloud/pkg/hostman/service"
func main() {
var srv = service.SHostService{}
srv.StartService()
}
+12 -2
View File
@@ -36,6 +36,9 @@ type Application struct {
defHandlerInfo SHandlerInfo
cors *Cors
middlewares []MiddlewareFunc
// record Http server for handle shotdown
server *http.Server
}
const (
@@ -326,13 +329,20 @@ func (app *Application) initServer(addr string) *http.Server {
}
func (app *Application) ListenAndServe(addr string) {
s := app.initServer(addr)
err := s.ListenAndServe()
app.server = app.initServer(addr)
err := app.server.ListenAndServe()
if err != nil {
log.Fatalf("ListAndServer fail: %s", err)
}
}
func (app *Application) ShowDown(ctx context.Context) error {
if app.server != nil {
return app.server.Shutdown(ctx)
}
return fmt.Errorf("Not init http server ??")
}
func (app *Application) ListenAndServeTLS(addr string, certFile, keyFile string) {
s := app.initServer(addr)
err := s.ListenAndServeTLS(certFile, keyFile)
+1
View File
@@ -0,0 +1 @@
package workmanager // import "yunion.io/x/onecloud/pkg/cloudcommon/workmanager"
+38
View File
@@ -0,0 +1,38 @@
package workmanager
import (
"sync/atomic"
"yunion.io/x/log"
)
type SWorkManager struct {
curCount int32
}
func (w *SWorkManager) add() {
atomic.AddInt32(&w.curCount, 1)
}
func (w *SWorkManager) done() {
atomic.AddInt32(&w.curCount, -1)
}
func (w *SWorkManager) RunTask(task func()) {
w.add()
go func() {
defer w.done()
task()
}()
}
func (w *SWorkManager) Stop() {
log.Infof("WorkManager To stop, wait for workers ...")
for w.curCount > 0 {
log.Warningf("Busy workers count %d, waiting stopped", w.curCount)
}
}
func NewWorkManger() *SWorkManager {
return &SWorkManager{}
}
+40 -12
View File
@@ -15,18 +15,11 @@ import (
type strDict map[string]string
type actionFunc func(context.Context, string, jsonutils.JSONObject) (interface{}, error)
var actionFuncs = map[string]actionFunc{
"create": doCreate,
"deploy": doDeploy,
"start": doStart,
"stop": doStop,
"monitor": doMonitor,
}
func AddGuestTaskHandler(prefix string, app *appsrv.Application) {
app.AddHandler("GET", "/servers/<sid>/status", auth.Authenticate(getStatus))
app.AddHandler("POST", "/servers/cpu-node-balance", auth.Authenticate(cpusetBalance))
app.AddHandler("POST", "/servers/<sid>/<action>", auth.Authenticate(guestActions))
app.AddHandler("DELETE", "/servers/<sid>", auth.Authenticate(deleteGuest))
}
func getStatus(ctx context.Context, w http.ResponseWriter, r *http.Request) {
@@ -36,7 +29,7 @@ func getStatus(ctx context.Context, w http.ResponseWriter, r *http.Request) {
}
func cpusetBalance(ctx context.Context, w http.ResponseWriter, r *http.Request) {
go guestManger.CpusetBalance(ctx)
wm.RunTask(func() { guestManger.CpusetBalance(ctx) })
responseOk(ctx, w)
}
@@ -58,12 +51,25 @@ func guestActions(ctx context.Context, w http.ResponseWriter, r *http.Request) {
}
}
func deleteGuest(ctx context.Context, w http.ResponseWriter, r *http.Request) {
params, _, body := appsrv.FetchEnv(ctx, w, r)
var sid = params["<sid>"]
var migrated = jsonutils.QueryBoolean(body, "migrated", false)
guest, err := guestManger.Delete(sid)
if err != nil {
response(ctx, w, err)
} else {
wm.RunTask(func() { guest.CleanGuest(ctx, migrated) })
response(ctx, w, map[string]bool{"delay_clean": true})
}
}
func doCreate(ctx context.Context, sid string, body jsonutils.JSONObject) (interface{}, error) {
err := guestManger.PrepareCreate(sid)
if err != nil {
return nil, httperrors.NewBadRequestError(err.Error())
}
go guestManger.DoDeploy(ctx, sid, body, true)
wm.RunTask(func() { guestManger.DoDeploy(ctx, sid, body, true) })
return nil, nil
}
@@ -73,7 +79,7 @@ func doDeploy(ctx context.Context, sid string, body jsonutils.JSONObject) (inter
}
func doStart(ctx context.Context, sid string, body jsonutils.JSONObject) (interface{}, error) {
// TODO
res, err := guestManger.Start(ctx, sid, body)
return nil, nil
}
@@ -96,8 +102,9 @@ func doMonitor(ctx context.Context, sid string, body jsonutils.JSONObject) (inte
var res = <-c
return strDict{"results": path.Join("\n", res)}, nil
}
} else {
return nil, httperrors.NewMissingParameterError("cmd")
}
return nil, httperrors.NewMissingParameterError("cmd")
}
func responseOk(ctx context.Context, w http.ResponseWriter) {
@@ -119,3 +126,24 @@ func response(ctx context.Context, w http.ResponseWriter, res interface{}) {
appsrv.SendStruct(w, res)
}
}
var actionFuncs = map[string]actionFunc{
"create": doCreate,
"deploy": doDeploy,
"start": doStart,
"stop": doStop,
"monitor": doMonitor,
"sync": doSync,
"suspend": doSuspend,
"snapshot": doSnapshot,
"delete-snapshot": doDeleteSnapshot,
"reload-disk-snapshot": doReloadDiskSnapshot,
"remove-statefile": doRemoveStatefile,
"io-throttle": doIoThrottle,
"src-prepare-migrate": doSrcPrepareMigrate,
"dest-prepare-migrate": doDestPrepareMigrate,
"live-migrate": doLiveMigrate,
"resume": doResume,
"start-nbd-server": doStartNbdServer,
"drive-mirror": doDriveMirror,
}
+56 -5
View File
@@ -14,6 +14,8 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/cloudcommon/httpclients"
"yunion.io/x/onecloud/pkg/cloudcommon/workmanager"
"yunion.io/x/onecloud/pkg/hostman"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/pkg/util/regutils"
)
@@ -27,7 +29,7 @@ type SGuestManager struct {
isLoaded bool
}
func NewSGuestManager(serversPath string) *SGuestManager {
func NewGuestManager(serversPath string) *SGuestManager {
manager := &SGuestManager{}
manager.ServersPath = serversPath
manager.Servers = make(map[string]*SKVMGuestInstance, 0)
@@ -81,7 +83,7 @@ func (m *SGuestManager) VerifyExistingGuests(pendingDelete bool) {
func (m *SGuestManager) OnVerifyExistingGuestsFail(err error, pendingDelete bool) {
log.Errorf("OnVerifyExistingGuestFail: %s, try again 30 seconds later", err.Error())
AddTimeout(30*time.Second, func() { m.VerifyExistingGuests(false) })
hostman.AddTimeout(30*time.Second, func() { m.VerifyExistingGuests(false) })
}
func (m *SGuestManager) OnVerifyExistingGuestsSucc(res jsonutils.JSONObject, pendingDelete bool) {
@@ -193,7 +195,7 @@ func (m *SGuestManager) Monitor(sid, cmd string, callback func(string)) error {
}
func (m *SGuestManager) DoDeploy(ctx context.Context, sid string, body jsonutils.JSONObject, isInit bool) {
// TODO
}
// delay cpuset balance
@@ -219,11 +221,49 @@ func (m *SGuestManager) Status(sid string) string {
}
}
var guestManger *SGuestManager
func (m *SGuestManager) Delete(sid string) (*SKVMGuestInstance, error) {
if guest, ok := m.Servers[sid]; ok {
delete(m.Servers, sid)
// 这里应该不需要append到deleted servers, 据观察 deleted servers没有用到
return guest, nil
} else {
return nil, httperrors.NewNotFoundError("Not found")
}
}
func (m *SGuestManager) Start(ctx context.Context, sid string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
if guest, ok := m.Servers[sid]; ok {
if desc, err := body.Get("desc"); err != nil {
// TODO
guest.SaveDesc(desc)
}
if guest.IsStopped() {
params, _ := body.Get("params")
// TODO
if err := guest.StartGuest(ctx, params); err != nil {
return nil, httperrors.NewBadRequestError("Failed to start server")
} else {
return jsonutils.NewDict(jsonutils.JSONPair{"vnc_port", jsonutils.NewInt(0)}), nil
}
} else {
vncPort := guest.GetVncPort()
if vncPort > 0 {
res := jsonutils.NewDict()
res.Set("vnc_port", jsonutils.NewInt(int64(vncPort)))
res.Set("is_running", jsonutils.JSONTrue)
return res, nil
} else {
return nil, httperrors.NewBadGatewayError("Seems started, but no VNC info")
}
}
} else {
return nil, httperrors.NewNotFoundError("Not found")
}
}
func initGuestManager(serversPath string) {
if guestManger == nil {
guestManger = NewSGuestManager(serversPath)
guestManger = NewGuestManager(serversPath)
}
}
@@ -238,3 +278,14 @@ func Stop() {
func Init(serversPath string) {
initGuestManager(serversPath)
}
func GetWorkManager() *workmanager.SWorkManager {
return wm
}
var guestManger *SGuestManager
var wm *workmanager.SWorkManager
func init() {
wm = workmanager.NewWorkManger()
}
+98 -7
View File
@@ -16,7 +16,9 @@ import (
"yunion.io/x/log"
"yunion.io/x/pkg/util/regutils"
"yunion.io/x/onecloud/pkg/appctx"
"yunion.io/x/onecloud/pkg/cloudcommon/httpclients"
"yunion.io/x/onecloud/pkg/hostman"
"yunion.io/x/onecloud/pkg/hostman/monitor"
"yunion.io/x/onecloud/pkg/hostman/options"
)
@@ -24,6 +26,7 @@ import (
const (
STATE_FILE_PREFIX = "STATEFILE"
MONITOR_PORT_BASE = 55900
MAX_TRY = 3
)
type SKVMGuestInstance struct {
@@ -161,9 +164,40 @@ func (s *SKVMGuestInstance) DirtyServerRequestStart() {
}
}
func (s *SKVMGuestInstance) AsyncScriptStart() {
// Must called in new goroutine
func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params *jsonutils.JSONDict) {
// TODO
// s.manager.RequestStartGuest()
// hostinof.instace().clean_deleted_ports
time.Sleep(100 * time.Millisecond)
var isStarted, tried, err = false, 0, nil
for !isStarted && tried < MAX_TRY {
tried += 1
vncPort := s.manager.GetFreeVncPort()
s.saveVncPort(vncPort)
params.Set("vnc_port", jsonutils.NewInt(vncPort))
s.saveScripts(params)
isStarted, err = s.scriptStart()
if !isStarted {
log.Errorf("Start VM failed: %s", err)
time.Sleep((1 << (tried - 1)) * time.Seconde)
} else {
log.Infof("VM started ...")
}
}
s.onAsyncScriptStart(ctx, isStarted, err)
}
func (s *SKVMGuestInstance) onAsyncScriptStart(ctx context.Context, isStarted bool, err error) {
if isStarted {
log.Infof("Async start server %s success!", s.GetName())
s.StartMonitor(ctx)
} else {
log.Infof("Async start server %s failed: %s!!!", s.GetName(), err)
if ctx != nil {
s.TaskFailed(ctx, fmt.Sprintf("Async start server failed: %s", err))
}
s.SyncStatus()
}
}
func (s *SKVMGuestInstance) ImportServer(pendingDelete bool) {
@@ -173,14 +207,14 @@ func (s *SKVMGuestInstance) ImportServer(pendingDelete bool) {
jsonutils.QueryBoolean(s.Desc, "is_slave", false) {
s.DirtyServerRequestStart()
} else {
s.AsyncScriptStart()
s.StartGuest(nil, nil)
}
return
}
if s.IsRunning() {
log.Infof("%s is running, pending_delete=%s", s.GetName(), pendingDelete)
if !pendingDelete {
s.StartMonitor(nil)
go s.StartMonitor(nil)
}
} else {
var action = "stopped"
@@ -196,6 +230,10 @@ func (s *SKVMGuestInstance) IsRunning() bool {
return s.GetPid() > 0
}
func (s *SKVMGuestInstance) IsStopped() bool {
return !s.IsRunning()
}
func (s *SKVMGuestInstance) IsSuspend() bool {
if !s.IsRunning() && len(s.ListStateFilePaths()) > 0 {
return true
@@ -217,9 +255,11 @@ func (s *SKVMGuestInstance) ListStateFilePaths() []string {
return nil
}
// Must called in new goroutine
func (s *SKVMGuestInstance) StartMonitor(ctx context.Context) {
// delay 100ms start monitor
AddTimeout(100*time.Millisecond, func() { s.delayStartMonitor(ctx) })
// delay 100ms start monitor // hostman.AddTimeout(100*time.Millisecond, func() { s.delayStartMonitor(ctx) })
time.Sleep(100 * time.Millisecond)
s.delayStartMonitor(ctx)
}
func (s *SKVMGuestInstance) delayStartMonitor(ctx context.Context) {
@@ -240,7 +280,19 @@ func (s *SKVMGuestInstance) onMonitorConnected(ctx context.Context) {
func (s *SKVMGuestInstance) onGetQemuVersion(ctx context.Context, version string) {
s.QemuVersion = version
log.Infof("Guest(%s) qemu version %s", s.Id, s.QemuVersion)
// TODO
if s.Desc.Contains("live_migrate_dest_port") && ctx != nil {
migratePort, _ := s.Desc.Get("live_migrate_dest_port")
body := jsonutils.NewDict(
jsonutils.JSONPair{"live_migrate_dest_port", migratePort})
s.TaskComplete(ctx, body)
} else if jsonutils.QueryBoolean(s.Desc, "is_slave", false) {
// TODO
} else if jsonutils.QueryBoolean(s.Desc, "is_master", false) && ctx == nil {
// TODO
} else {
// TODO
s.DoResumeTask(ctx)
}
}
func (s *SKVMGuestInstance) onMonitorDisConnect(err error) {
@@ -288,3 +340,42 @@ func (s *SKVMGuestInstance) SyncStatus() {
}
httpclients.GetDefaultComputeClient().UpdateServerStatus(s.GetId(), status)
}
func (s *SKVMGuestInstance) TaskFailed(ctx context.Context, reason string) error {
if taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID); taskId != nil {
httpclients.GetDefaultComputeClient().TaskFail(ctx, taskId.(string), reason)
return nil
} else {
log.Errorln("Reqeuest task failed missing task id, with reason(%s)", reason)
return fmt.Errorf("Reqeuest task failed missing task id")
}
}
func (s *SKVMGuestInstance) TaskComplete(ctx context.Context, data jsonutils.JSONObject) error {
if taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID); taskId != nil {
httpclients.GetDefaultComputeClient().TaskComplete(ctx, taskId.(string), data, 0)
return nil
} else {
log.Errorln("Reqeuest task complete missing task id")
return fmt.Errorf("Reqeuest task complete missing task id")
}
}
func (s *SKVMGuestInstance) SaveDesc(desc jsonutils.JSONObject) error {
// TODO
// bw_info = self._get_bw_info()
// netmon_info = self._get_netmon_info()
s.Desc = desc.(*jsonutils.JSONDict)
if err := hostman.FilePutContents(s.GetDescFilePath(), desc.String()); err != nil {
log.Errorln(err)
}
// TODO
// self._update_bw_limit(bw_info)
// self._update_netmon_nic(netmon_info)
}
func (s *SKVMGuestInstance) StartGuest(ctx context.Context, params jsonutils.JSONObject) {
wm.RunTask(func() {
s.asyncScriptStart(ctx, params)
})
}
+7
View File
@@ -0,0 +1,7 @@
package guestman
import "yunion.io/x/jsonutils"
func DoTask(guest *SKVMGuestInstance, kwargs jsonutils.JSONObject) {
}
-38
View File
@@ -1,39 +1 @@
package guestman
import (
"io/ioutil"
"time"
)
// timer utils
func AddTimeout(second time.Duration, callback func()) {
go func() {
<-time.NewTimer(second).C
callback()
}()
}
/*
func PathNotExists(path string) bool {
if _, err := os.Stat(path); os.IsNotExist(err) {
return true
}
return false
}
func PathExists(path string) bool {
if _, err := os.Stat(path); !os.IsNotExist(err) {
return true
}
return false
}
*/
func FileGetContents(file string) (string, error) {
content, err := ioutil.ReadFile(file)
if err != nil {
return "", err
}
return string(content), nil
}
+27 -18
View File
@@ -9,24 +9,10 @@ import (
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/cloudcommon/qemutils"
"yunion.io/x/onecloud/pkg/hostman"
"yunion.io/x/onecloud/pkg/hostman/options"
)
func NewHostInfo() *SHostInfo {
cpu := DetectCpuInfo()
// memory := DetectMemoryInfo()
return &SHostInfo{
Cpu: cpu,
}
}
var hostInfo *SHostInfo
func Init() error {
hostInfo = NewHostInfo()
return hostInfo.Start()
}
type SHostInfo struct {
isRegistered bool
@@ -101,7 +87,7 @@ func (h *SHostInfo) prepareEnv() error {
ioParams["queue/iosched/group_idle"] = "0"
ioParams["queue/iosched/quantum"] = "32"
}
ChangeAllBlkdevsParams(ioParams)
hostman.ChangeAllBlkdevsParams(ioParams)
_, err = exec.Command("modprobe", "tun").Output()
if err != nil {
return fmt.Errorf("Failed to activate tun/tap device")
@@ -217,7 +203,7 @@ func (h *SHostInfo) EnableNativeHugepages() error {
// TODO
// h.Memory 还未实现
preAllocPagesNum := h.GetMemory/h.Memory.GetHugepagesizeMb() + 1
cmd := CommandWithTimeout(1, "sh", "-c", fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", preAllocPagesNum))
cmd := hostman.CommandWithTimeout(1, "sh", "-c", fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", preAllocPagesNum))
_, err := cmd.Output()
if err != nil {
log.Errorln(err)
@@ -236,7 +222,7 @@ func (h *SHostInfo) setSysConfig(path, val string) bool {
if _, err := os.Stat(path); !os.IsNotExist(err) {
oval, _ := ioutil.ReadFile(path)
if string(oval) != val {
err = FilePutContents(path, val, false)
err = hostman.FilePutContents(path, val, false)
if err == nil {
return true
}
@@ -281,3 +267,26 @@ func (h *SHostInfo) resetIptables() error {
}
return nil
}
func (h *SHostInfo) StartRegister() {
// TODO
}
func NewHostInfo() *SHostInfo {
cpu := DetectCpuInfo()
// memory := DetectMemoryInfo()
return &SHostInfo{
Cpu: cpu,
}
}
var hostInfo *SHostInfo
func Init() error {
hostInfo = NewHostInfo()
return hostInfo.Start()
}
func Instance() *SHostInfo {
return hostInfo
}
+13 -14
View File
@@ -1,6 +1,7 @@
package service
import (
"context"
"os"
"yunion.io/x/log"
@@ -8,6 +9,7 @@ import (
"yunion.io/x/onecloud/pkg/cloudcommon"
"yunion.io/x/onecloud/pkg/cloudcommon/service"
"yunion.io/x/onecloud/pkg/hostman/guestman"
"yunion.io/x/onecloud/pkg/hostman/hostinfo"
"yunion.io/x/onecloud/pkg/hostman/options"
)
@@ -16,8 +18,6 @@ type SHostService struct {
}
func (host *SHostService) StartService() {
host.TrapSignals(host.quitSignalHandler)
cloudcommon.ParseOptions(&options.HostOptions, &options.HostOptions.Options, os.Args, "host.conf")
// Hostinfo.Init()
// Firewall.Init()
@@ -26,26 +26,25 @@ func (host *SHostService) StartService() {
cloudcommon.InitAuth(&options.HostOptions.Options, func() {
log.Infof("Auth complete!!")
// TODO
// hostinfo.instance().start_register
// 应该在register 成功后注册handler
// 上报guest信息,即guestman
// hostinfo.startregisters
hostinfo.Instance().StartRegister()
guestman.Init(options.HostOptions.ServersPath)
close(c)
})
guestman.Init()
app := cloudcommon.InitApp(&o.Options.Options)
app := cloudcommon.InitApp(&options.HostOptions.Options)
host.TrapSignals(func() { host.quitSignalHandler(app) })
host.InitHandlers(app)
<-c // wait host info registered
cloudcommon.ServeForever(app, &options.HostOptions)
}
func (host *SHostService) quitSignalHandler() {
func (host *SHostService) quitSignalHandler(app *appsrv.Application) {
// TODO
/*
cloud/yunion/server/clouds/common/handler/__init__.py -> stop()
1. delay process
2. work manager
*/
/* cloud/yunion/server/clouds/common/handler/__init__.py -> stop() */
err := app.ShowDown(context.Background())
if err != nil {
log.Errorln(err.Error())
}
guestman.GetWorkManager().Stop()
}
func (host *SHostService) initHandlers(app *appsrv.Application) {
@@ -1,4 +1,4 @@
package hostinfo
package hostman
import (
"fmt"
@@ -7,6 +7,7 @@ import (
"os/exec"
"path"
"strings"
"time"
"yunion.io/x/log"
)
@@ -74,3 +75,36 @@ func ChangeBlkdevParameter(dev, key, value string) {
log.Infof("Set %s of %s to %s", key, dev, value)
}
}
// timer utils
func AddTimeout(second time.Duration, callback func()) {
go func() {
<-time.NewTimer(second).C
callback()
}()
}
/*
func PathNotExists(path string) bool {
if _, err := os.Stat(path); os.IsNotExist(err) {
return true
}
return false
}
func PathExists(path string) bool {
if _, err := os.Stat(path); !os.IsNotExist(err) {
return true
}
return false
}
*/
func FileGetContents(file string) (string, error) {
content, err := ioutil.ReadFile(file)
if err != nil {
return "", err
}
return string(content), nil
}