From 9251c6b45b9886b16f29f2b74909b3d4fcb940d2 Mon Sep 17 00:00:00 2001
From: Yousong Zhou <zhouyousong@yunionyun.com>
Date: Sat, 29 Dec 2018 08:33:49 +0000
Subject: [PATCH] =?UTF-8?q?lbagent:=20=E5=A2=9E=E5=8A=A0haproxy=20reload?=
 =?UTF-8?q?=E5=81=A5=E5=A3=AE=E6=80=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

HAProxy小概率出现stats socket和stats url未监听到的情况，怀疑是00-haproxy.cfg
被提前删除造成，导致后续的-x haproxy.sock配置重加载失败

健壮性增强，reload失败时，杀掉旧进程pid，重新启动新的haproxy
---
 pkg/lbagent/haproxy.go | 51 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 45 insertions(+), 6 deletions(-)

diff --git a/pkg/lbagent/haproxy.go b/pkg/lbagent/haproxy.go
index 1c1f895aa0..0e8b1625d3 100644
--- a/pkg/lbagent/haproxy.go
+++ b/pkg/lbagent/haproxy.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	"yunion.io/x/log"
 	aggrerrors "yunion.io/x/pkg/util/errors"
@@ -242,21 +243,59 @@ func (h *HaproxyHelper) reloadHaproxy(ctx context.Context) error {
 		"-f", h.haproxyConfD(),
 	}
 	proc := agentutils.ReadPidFile(pidFile)
-	if proc != nil {
-		args = append(args, "-sf", fmt.Sprintf("%d", proc.Pid))
+	if proc == nil {
+		log.Infof("starting haproxy")
+		return h.runCmd(args)
+	}
+
+	{
+		// try reload
+		args_ := make([]string, len(args))
+		copy(args_, args)
+		args_ = append(args_, "-sf", fmt.Sprintf("%d", proc.Pid))
 		{
 			statsSocket := h.haproxyStatsSocketFile()
 			if fi, err := os.Stat(statsSocket); err == nil && fi.Mode()&os.ModeSocket != 0 {
-				args = append(args, "-x", statsSocket)
+				args_ = append(args_, "-x", statsSocket)
 			} else {
 				log.Warningf("stats socket %s not found", statsSocket)
 			}
 		}
 		log.Infof("reloading haproxy")
-	} else {
-		log.Infof("starting haproxy")
+		err := h.runCmd(args_)
+		if err == nil {
+			return nil
+		}
+		log.Errorf("reloading haproxy: %s", err)
+	}
+	{
+		// reload failed
+		// kill the old
+		log.Errorf("killing old haproxy %d", proc.Pid)
+		proc.Signal(syscall.SIGKILL)
+		killed := false
+	loop:
+		for {
+			timeout := time.NewTimer(3 * time.Second)
+			ticker := time.NewTicker(10 * time.Millisecond)
+			defer ticker.Stop()
+			defer timeout.Stop()
+			select {
+			case <-ticker.C:
+				if err := proc.Signal(syscall.Signal(0)); err != nil {
+					killed = true
+					break loop
+				}
+			case <-timeout.C:
+				break loop
+			}
+		}
+		if !killed {
+			return fmt.Errorf("failed killing haproxy %d", proc.Pid)
+		}
+		log.Infof("restarting haproxy")
+		return h.runCmd(args)
 	}
-	return h.runCmd(args)
 }
 
 func (h *HaproxyHelper) gobetweenConf() string {