feat(host): aware of kubelet eviction config

pkg/hostman/host_services.go

@@ -81,12 +81,12 @@ func (host *SHostService) RunService() {
 
 	hostInstance := hostinfo.Instance()
 	if err := hostInstance.Init(); err != nil {
-		log.Fatalf(err.Error())
+		log.Fatalf("Host instance init error: %v", err)
 	}
 
 	deployclient.Init(options.HostOptions.DeployServerSocketPath)
 	if err := storageman.Init(hostInstance); err != nil {
-		log.Fatalf(err.Error())
+		log.Fatalf("Storage manager init error: %v", err)
 	}
 
 	var guestChan chan struct{}
@@ -95,7 +95,7 @@ func (host *SHostService) RunService() {
 		log.Infof("Auth complete!!")
 
 		if err := host.initEtcdConfig(); err != nil {
-			log.Fatalln(err)
+			log.Fatalln("Init etcd config: %v", err)
 		}
 
 		hostInstance.StartRegister(2, func() {

pkg/hostman/hostinfo/hostinfo.go

@@ -43,6 +43,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge"
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
@@ -72,6 +73,8 @@ type SHostInfo struct {
 	Mem     *SMemory
 	sysinfo *SSysInfo
 
+	kubeletConfig kubelet.KubeletConfig
+
 	isInit          bool
 	enableHugePages bool
 	onHostDown      string
@@ -158,7 +161,7 @@ func (h *SHostInfo) IsHugepagesEnabled() bool {
  */
 func (h *SHostInfo) Init() error {
 	if err := h.prepareEnv(); err != nil {
-		return err
+		return errors.Wrap(err, "Prepare environment")
 	}
 
 	log.Infof("Start detectHostInfo")
@@ -317,7 +320,7 @@ func (h *SHostInfo) parseConfig() error {
 
 func (h *SHostInfo) prepareEnv() error {
 	if err := h.fixPathEnv(); err != nil {
-		return err
+		return errors.Wrap(err, "Fix path environment")
 	}
 	if options.HostOptions.ReportInterval > 300 {
 		return fmt.Errorf("Option report_interval must no longer than 5 min")
@@ -330,7 +333,7 @@ func (h *SHostInfo) prepareEnv() error {
 
 	_, err = procutils.NewCommand("ethtool", "-h").Output()
 	if err != nil {
-		return fmt.Errorf("Ethtool not installed")
+		return errors.Wrap(err, "Execute 'ethtool -h'")
 	}
 
 	ioParams := make(map[string]string, 0)
@@ -346,11 +349,11 @@ func (h *SHostInfo) prepareEnv() error {
 	fileutils2.ChangeAllBlkdevsParams(ioParams)
 	_, err = procutils.NewRemoteCommandAsFarAsPossible("modprobe", "tun").Output()
 	if err != nil {
-		return fmt.Errorf("Failed to activate tun/tap device")
+		return errors.Wrap(err, "Failed to activate tun/tap device")
 	}
 	output, err = procutils.NewRemoteCommandAsFarAsPossible("modprobe", "vhost_net").Output()
 	if err != nil {
-		log.Errorf("modprobe error: %s", output)
+		log.Warningf("modprobe vhost_net error: %s", output)
 	}
 	if !options.HostOptions.DisableSetCgroup {
 		if !cgrouputils.Init() {
@@ -495,7 +498,16 @@ func (h *SHostInfo) GetMemory() (int, error) {
 	if options.HostOptions.HugepagesOption == "native" {
 		return h.Mem.GetHugepageTotal()
 	}
-	return h.Mem.Total, nil // - options.reserved_memory
+	total := h.Mem.Total
+	if h.kubeletConfig != nil {
+		memThreshold := h.kubeletConfig.GetEvictionConfig().GetHard().GetMemoryAvailable()
+		memBytes, _ := memThreshold.Value.Quantity.AsInt64()
+		memMb := int(memBytes / 1024 / 1024)
+		subMem := total - memMb
+		log.Infof("Get total memory %d, kubelet memory threshold subtracted: (%d - %d)", subMem, total, memMb)
+		total = subMem
+	}
+	return total, nil // - options.reserved_memory
 }
 
 func (h *SHostInfo) getCurrentHugepageNr() (int64, error) {
@@ -1394,11 +1406,11 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) {
 					params.Set("is_root_partiton", jsonutils.JSONTrue)
 					_, err := modules.Hoststorages.Update(h.GetSession(), h.HostId, storageId, nil, params)
 					if err != nil {
-						h.onFail(err)
+						h.onFail(errors.Wrapf(err, "Update host storage %s with params %s", storageId, params))
 					}
 				}
 				if err := storage.SetStorageInfo(storageId, storageName, storageConf); err != nil {
-					h.onFail(err)
+					h.onFail(errors.Wrapf(err, "Set storage info %s/%s/%s", storageId, storageName, storageConf))
 				}
 			} else {
 				// XXX hack: storage type baremetal is a converted host，reserve storage
@@ -1419,11 +1431,11 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) {
 func (h *SHostInfo) uploadStorageInfo() {
 	for _, s := range storageman.GetManager().Storages {
 		if err := s.SetStorageInfo(s.GetId(), s.GetStorageName(), s.GetStorageConf()); err != nil {
-			h.onFail(err)
+			h.onFail(errors.Wrapf(err, "Upload storage %s info with config %s", s.GetStorageName(), s.GetStorageConf()))
 		}
 		res, err := s.SyncStorageInfo()
 		if err != nil {
-			h.onFail(err)
+			h.onFail(errors.Wrapf(err, "Sync storage %s info", s.GetStorageName()))
 		} else {
 			h.onSyncStorageInfoSucc(s, res)
 		}
@@ -1750,6 +1762,10 @@ func (h *SHostInfo) IsX8664() bool {
 	return h.GetCpuArchitecture() == apis.OS_ARCH_X86_64
 }
 
+func (h *SHostInfo) GetKubeletConfig() kubelet.KubeletConfig {
+	return h.kubeletConfig
+}
+
 func NewHostInfo() (*SHostInfo, error) {
 	var res = new(SHostInfo)
 	res.sysinfo = &SSysInfo{}
@@ -1779,6 +1795,17 @@ func NewHostInfo() (*SHostInfo, error) {
 	res.IsRegistered = make(chan struct{})
 	res.SysError = make(map[string]string)
 	res.SysWarning = make(map[string]string)
+
+	if !options.HostOptions.DisableProbeKubelet {
+		kubeletDir := options.HostOptions.KubeletRunDirectory
+		kubeletConfig, err := kubelet.NewKubeletConfigByDirectory(kubeletDir)
+		if err != nil {
+			return nil, errors.Wrapf(err, "New kubelet config by dir: %s", kubeletDir)
+		}
+		res.kubeletConfig = kubeletConfig
+		log.Infof("Get kubelet container image Fs: %s, eviction config: %s", res.kubeletConfig.GetImageFs(), res.kubeletConfig.GetEvictionConfig())
+	}
+
 	return res, nil
 }
 
@@ -1789,7 +1816,7 @@ func Instance() *SHostInfo {
 		var err error
 		hostInfo, err = NewHostInfo()
 		if err != nil {
-			log.Fatalln(err)
+			log.Fatalf("NewHostInfo: %s", err)
 		}
 	}
 	return hostInfo

pkg/hostman/hostutils/hostutils.go

@@ -26,6 +26,7 @@ import (
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/workmanager"
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -54,6 +55,8 @@ type IHost interface {
 	GetIsolatedDeviceManager() *isolated_device.IsolatedDeviceManager
 
 	SyncRootPartitionUsedCapacity() error
+
+	GetKubeletConfig() kubelet.KubeletConfig
 }
 
 func GetComputeSession(ctx context.Context) *mcclient.ClientSession {

pkg/hostman/hostutils/kubelet/doc.go

@@ -0,0 +1 @@
+package kubelet // import "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet"

pkg/hostman/hostutils/kubelet/docker.go

@@ -0,0 +1,47 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubelet
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type DockerInfo struct {
+	ID            string `json:"ID"`
+	Driver        string `json:"Driver"`
+	DockerRootDir string `json:"DockerRootDir"`
+}
+
+func GetDockerInfoByRemote() (*DockerInfo, error) {
+	content, err := procutils.NewRemoteCommandAsFarAsPossible("docker", "info", "--format", "{{json .}}").Output()
+	if err != nil {
+		return nil, errors.Wrap(err, "Run command 'docker info'")
+	}
+
+	obj, err := jsonutils.Parse(content)
+	if err != nil {
+		return nil, errors.Wrap(err, "Parse docker info to json")
+	}
+
+	info := new(DockerInfo)
+	if err := obj.Unmarshal(info); err != nil {
+		return nil, errors.Wrap(err, "Unmarshal docker info")
+	}
+
+	return info, nil
+}

pkg/hostman/hostutils/kubelet/eviction/doc.go

@@ -0,0 +1 @@
+package eviction // import "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet/eviction"

pkg/hostman/hostutils/kubelet/eviction/eviction.go

@@ -0,0 +1,264 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package eviction
+
+import (
+	"encoding/json"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+)
+
+// Signal defines a signal that can trigger eviction of pods on a node.
+type Signal string
+
+const (
+	// SignalMemoryAvailable is memory available (i.e. capacity - workingSet), in bytes.
+	SignalMemoryAvailable Signal = "memory.available"
+	// SignalNodeFsAvailable is amount of storage available on filesystem that kubelet uses for volumes, daemon logs, etc.
+	SignalNodeFsAvailable Signal = "nodefs.available"
+	// SignalNodeFsInodesFree is amount of storage available on filesystem that container runtime uses for storing images and container writable layers.
+	SignalNodeFsInodesFree Signal = "nodefs.inodesFree"
+	// SignalImageFsAvailable is amount of storage available on filesystem that container runtime uses for storing images and container writable layers.
+	SignalImageFsAvailable Signal = "imagefs.available"
+	// SignalImageFsInodesFree is amount of inodes available on filesystem that container runtime uses for storing images and container writable layers.
+	SignalImageFsInodesFree Signal = "imagefs.inodesFree"
+	// SignalAllocatableMemoryAvailable is amount of memory available for pod allocation (i.e. allocatable - workingSet (of pods), in bytes)
+	// SignalAllocatableMemoryAvailable Signal = "allocatableMemory.available"
+	// SignalPIDAvailable is amount of PID available for pod allocation
+	SignalPIDAvailable Signal = "pid.available"
+)
+
+var (
+	// DefaultEvictionHard includes default options for kubelet hard eviction
+	// ref: https://github.com/kubernetes/kubernetes/blob/ec39cc2eafffa51b3267e3bd64fbd2598c0db94d/pkg/kubelet/apis/config/v1beta1/defaults_linux.go#L21
+	DefaultEvictionHard = map[string]string{
+		string(SignalMemoryAvailable):  "100Mi",
+		string(SignalNodeFsAvailable):  "10%",
+		string(SignalNodeFsInodesFree): "5%",
+		string(SignalImageFsAvailable): "15%",
+	}
+)
+
+type ThresholdMap map[Signal]*Threshold
+
+func (m ThresholdMap) GetMemoryAvailable() *Threshold {
+	return m[SignalMemoryAvailable]
+}
+
+func (m ThresholdMap) GetNodeFsAvailable() *Threshold {
+	return m[SignalNodeFsAvailable]
+}
+
+func (m ThresholdMap) GetNodeFsInodesFree() *Threshold {
+	return m[SignalNodeFsInodesFree]
+}
+
+func (m ThresholdMap) GetImageFsAvailable() *Threshold {
+	return m[SignalImageFsAvailable]
+}
+
+// Config holds information about how eviction is configured.
+type Config interface {
+	GetHard() ThresholdMap
+	String() string
+}
+
+// config implements Config interface
+type config struct {
+	// hard holds configuration of hardThresholds
+	hard ThresholdMap
+}
+
+type configContent struct {
+	// Map of signal names to quantities that defines hard eviction thresholds. For example: {"memory.available": "300Mi"}
+	EvictionHard map[string]string `json:"evictionHard"`
+	// Map of signal names to quantities that defines soft eviction thresholds. For example: {"memory.available": "300Mi"}
+	// EvictionSoft map[string]string `json:"evictionSoft"`
+}
+
+func NewConfig(yamlContent []byte) (Config, error) {
+	obj, err := jsonutils.ParseYAML(string(yamlContent))
+	if err != nil {
+		return nil, errors.Wrapf(err, "Parse yaml content %q", yamlContent)
+	}
+
+	content := new(configContent)
+	if err := obj.Unmarshal(content); err != nil {
+		return nil, errors.Wrap(err, "Unmarshal eviction content")
+	}
+
+	hardThresholds, err := parseThresholdConfig(content.EvictionHard)
+	if err != nil {
+		return nil, errors.Wrap(err, "Parse hard thresholds")
+	}
+
+	return &config{
+		hard: hardThresholds,
+	}, nil
+}
+
+func (c *config) GetHard() ThresholdMap {
+	return c.hard
+}
+
+func (c *config) String() string {
+	out := map[string]interface{}{
+		"evictionHard": c.hard,
+	}
+
+	bytes, err := json.Marshal(out)
+	if err != nil {
+		log.Errorf("Marshal eviction config error: %v", err)
+	}
+	return string(bytes)
+}
+
+func parseThresholdConfig(evictionHard map[string]string) (map[Signal]*Threshold, error) {
+	results := map[Signal]*Threshold{}
+	hardThresholds, err := parseHardThresholdStatements(evictionHard)
+	if err != nil {
+		return nil, errors.Wrap(err, "Parse hard threshold")
+	}
+	for _, r := range hardThresholds {
+		results[r.Signal] = r
+	}
+	return results, nil
+}
+
+func parseHardThresholdStatements(statements map[string]string) ([]*Threshold, error) {
+	results := []*Threshold{}
+
+	for _, signal := range []Signal{
+		SignalMemoryAvailable,
+		SignalNodeFsAvailable,
+		SignalNodeFsInodesFree,
+		SignalImageFsAvailable,
+		SignalImageFsInodesFree,
+		SignalPIDAvailable,
+	} {
+		val, ok := statements[string(signal)]
+		if !ok {
+			// try get from default setting
+			val, ok = DefaultEvictionHard[string(signal)]
+			if !ok {
+				continue
+			}
+		}
+		result, err := parseThresholdStatement(signal, val)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Parse signal %q with val %q", signal, val)
+		}
+		if result == nil {
+			continue
+		}
+		results = append(results, result)
+	}
+	return results, nil
+}
+
+func parseThresholdStatement(signal Signal, val string) (*Threshold, error) {
+	operator, ok := OpForSignal[signal]
+	if !ok {
+		return nil, errors.Errorf("Unsupported signal %q", signal)
+	}
+	if strings.HasSuffix(val, "%") {
+		// ignore 0% and 100%
+		if val == "0%" || val == "100%" {
+			return nil, nil
+		}
+		percentage, err := parsePercentage(val)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Parse val %q to percentage", val)
+		}
+		if percentage < 0 {
+			return nil, errors.Errorf("Eviction percentage threshold %q must be >= 0%%: %q", signal, val)
+		}
+		if percentage > 100 {
+			return nil, errors.Errorf("Eviction percentage threshold %q must be <= 100%%: %q", signal, val)
+		}
+		return &Threshold{
+			Signal:   signal,
+			Operator: operator,
+			Value: ThresholdValue{
+				Percentage: percentage,
+			},
+		}, nil
+	}
+	quantity, err := resource.ParseQuantity(val)
+	if err != nil {
+		return nil, err
+	}
+	if quantity.Sign() < 0 || quantity.IsZero() {
+		return nil, errors.Errorf("Eviction threshold %q must be positive: %s", signal, &quantity)
+	}
+	return &Threshold{
+		Signal:   signal,
+		Operator: operator,
+		Value: ThresholdValue{
+			Quantity: &quantity,
+		},
+	}, nil
+}
+
+func parsePercentage(input string) (float32, error) {
+	val, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32)
+	if err != nil {
+		return 0, err
+	}
+	return float32(val) / 100, nil
+}
+
+// ThresholdOperator is the operator used to express a Threshold.
+type ThresholdOperator string
+
+const (
+	// OpLessThan is the operator that expresses a less than operator.
+	OpLessThan ThresholdOperator = "LessThan"
+)
+
+// OpForSignal maps Signals to ThresholdOperators.
+// Today, the only supported operator is "LessThan".
+var OpForSignal = map[Signal]ThresholdOperator{
+	SignalMemoryAvailable:   OpLessThan,
+	SignalNodeFsAvailable:   OpLessThan,
+	SignalNodeFsInodesFree:  OpLessThan,
+	SignalImageFsAvailable:  OpLessThan,
+	SignalImageFsInodesFree: OpLessThan,
+	SignalPIDAvailable:      OpLessThan,
+}
+
+// Threshold defines a metric for when eviction should occur.
+type Threshold struct {
+	// Signal defines the entity that was measured.
+	Signal Signal
+	// Operator represents a relationship of a signal to a value.
+	Operator ThresholdOperator
+	// Value is the threshold the resource is evaluated against.
+	Value ThresholdValue
+}
+
+// ThresholdValue is a value holder that abstracts literal versus percentage based quantity
+type ThresholdValue struct {
+	// Quantity is a quantity associated with the signal
+	Quantity *resource.Quantity
+	// Percentage represents the usage percentage over the total resource
+	Percentage float32
+}

pkg/hostman/hostutils/kubelet/eviction/eviction_test.go

@@ -0,0 +1,91 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package eviction
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewConfig(t *testing.T) {
+	type tCase struct {
+		name       string
+		input      string
+		assertFunc func(*testing.T, Config) error
+	}
+
+	cases := []tCase{
+		{
+			name:  "Should has default config with empty input",
+			input: ``,
+			assertFunc: func(t *testing.T, config Config) error {
+				memBytes, _ := config.GetHard().GetMemoryAvailable().Value.Quantity.AsInt64()
+				assert.Equal(t, int64(1024*1024*100), memBytes)
+				assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsAvailable().Value.Percentage)
+				assert.Equal(t, float32(0.05), config.GetHard().GetNodeFsInodesFree().Value.Percentage)
+				assert.Equal(t, float32(0.15), config.GetHard().GetImageFsAvailable().Value.Percentage)
+				return nil
+			},
+		},
+		{
+			name: "With all config",
+			input: `
+evictionHard:
+  imagefs.available: 25%
+  memory.available: 1024Mi
+  nodefs.available: 15%
+  nodefs.inodesFree: 10%`,
+			assertFunc: func(t *testing.T, config Config) error {
+				memBytes, _ := config.GetHard().GetMemoryAvailable().Value.Quantity.AsInt64()
+				assert.Equal(t, int64(1024*1024*1024), memBytes)
+				assert.Equal(t, float32(0.25), config.GetHard().GetImageFsAvailable().Value.Percentage)
+				assert.Equal(t, float32(0.15), config.GetHard().GetNodeFsAvailable().Value.Percentage)
+				assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsInodesFree().Value.Percentage)
+				return nil
+			},
+		},
+		{
+			name: "Mixed config with default",
+			input: `
+evictionHard:
+  imagefs.available: 5%
+  nodefs.inodesFree: 10%`,
+			assertFunc: func(t *testing.T, config Config) error {
+				memBytes, _ := config.GetHard().GetMemoryAvailable().Value.Quantity.AsInt64()
+				assert.Equal(t, int64(1024*1024*100), memBytes)
+				assert.Equal(t, float32(0.05), config.GetHard().GetImageFsAvailable().Value.Percentage)
+				assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsAvailable().Value.Percentage)
+				assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsInodesFree().Value.Percentage)
+				return nil
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(st *testing.T) {
+			if config, err := NewConfig([]byte(tc.input)); err != nil {
+				st.Errorf("[%s] NewConfig error: %v", tc.name, err)
+			} else {
+				str, _ := json.MarshalIndent(config, "", "  ")
+				st.Logf("%s", str)
+				if err := tc.assertFunc(st, config); err != nil {
+					st.Errorf("[%s] assert error: %v", tc.name, err)
+				}
+			}
+		})
+	}
+}

pkg/hostman/hostutils/kubelet/kubelet.go

@@ -0,0 +1,124 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubelet
+
+import (
+	"path"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet/eviction"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+const (
+	KubeletConfigurationFileName = "config.yaml"
+)
+
+// KubeletConfig is a interface abstracts manipulation of kubelet run directory.
+type KubeletConfig interface {
+	HasDedicatedImageFs() bool
+	GetNodeFsDevice() string
+	GetImageFsDevice() string
+	GetImageFs() string
+	GetEvictionConfig() eviction.Config
+}
+
+// kubeletConfig implements KubeletRunDirectory interface.
+type kubeletConfig struct {
+	config         jsonutils.JSONObject
+	dockerInfo     *DockerInfo
+	evictionConfig eviction.Config
+	nodeFsDevice   string
+	imageFsDevice  string
+}
+
+func NewKubeletConfigByDirectory(dir string) (KubeletConfig, error) {
+	configFile := path.Join(dir, KubeletConfigurationFileName)
+
+	content, err := procutils.NewRemoteCommandAsFarAsPossible("cat", configFile).Output()
+	if err != nil {
+		return nil, errors.Wrapf(err, "Read config file %s", configFile)
+	}
+
+	dockerInfo, err := GetDockerInfoByRemote()
+	if err != nil {
+		return nil, errors.Wrap(err, "Get docker info")
+	}
+
+	return newKubeletConfig(content, dockerInfo)
+}
+
+func newKubeletConfig(yamlConfig []byte, dockerInfo *DockerInfo) (KubeletConfig, error) {
+	obj, err := jsonutils.ParseYAML(string(yamlConfig))
+	if err != nil {
+		return nil, errors.Wrapf(err, "Parse yaml content %s", yamlConfig)
+	}
+
+	evictionConfig, err := eviction.NewConfig(yamlConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "New eviction config")
+	}
+
+	imageFsDev, err := GetDirectoryMountDevice(dockerInfo.DockerRootDir)
+	if err != nil {
+		return nil, errors.Wrap(err, "Find docker root directory device")
+	}
+
+	nodeFsDev, err := GetDirectoryMountDevice("/")
+	if err != nil {
+		return nil, errors.Wrap(err, "Find node FS directory device")
+	}
+
+	k := &kubeletConfig{
+		config:         obj,
+		dockerInfo:     dockerInfo,
+		evictionConfig: evictionConfig,
+		nodeFsDevice:   nodeFsDev,
+		imageFsDevice:  imageFsDev,
+	}
+
+	return k, nil
+}
+
+func GetDirectoryMountDevice(dirPath string) (string, error) {
+	content, err := procutils.NewRemoteCommandAsFarAsPossible("findmnt", "-n", "-o", "SOURCE", "--target", dirPath).Output()
+	if err != nil {
+		return "", errors.Wrapf(err, "Find directory %q mount source device", dirPath)
+	}
+	return strings.TrimSpace(string(content)), nil
+}
+
+func (k *kubeletConfig) GetNodeFsDevice() string {
+	return k.nodeFsDevice
+}
+
+func (k *kubeletConfig) GetImageFsDevice() string {
+	return k.imageFsDevice
+}
+
+func (k *kubeletConfig) HasDedicatedImageFs() bool {
+	return k.imageFsDevice != k.nodeFsDevice
+}
+
+func (k *kubeletConfig) GetImageFs() string {
+	return k.dockerInfo.DockerRootDir
+}
+
+func (k *kubeletConfig) GetEvictionConfig() eviction.Config {
+	return k.evictionConfig
+}

pkg/hostman/options/options.go

@@ -140,6 +140,9 @@ type SHostOptions struct {
 
 	SyncStorageInfoDurationSecond int  `help:"sync storage size duration, unit is second" default:"60"`
 	StartHostIgnoreSysError       bool `help:"start host agent ignore sys error" default:"false"`
+
+	DisableProbeKubelet bool   `help:"Disable probe kubelet config" default:"false"`
+	KubeletRunDirectory string `help:"Kubelet config file path" default:"/var/lib/kubelet"`
 }
 
 var (

pkg/hostman/storageman/core.go

@@ -29,6 +29,7 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/storageutils"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -73,7 +74,7 @@ func NewStorageManager(host hostutils.IHost) (*SStorageManager, error) {
 				allFull = false
 			}
 		} else {
-			log.Errorf("storage %s not accessible", s.Path)
+			log.Errorf("storage %s not accessible error: %v", s.Path, err)
 		}
 	}
 
@@ -122,6 +123,10 @@ func (s *SStorageManager) GetMediumType() string {
 	return s.host.GetMediumType()
 }
 
+func (s *SStorageManager) GetKubeletConfig() kubelet.KubeletConfig {
+	return s.host.GetKubeletConfig()
+}
+
 func (s *SStorageManager) getLeasedUsedLocalStorage(cacheDir string, limit int) (string, error) {
 	var (
 		maxFree    int

pkg/hostman/storageman/storage_local.go

@@ -17,6 +17,7 @@ package storageman
 import (
 	"context"
 	"fmt"
+	"math"
 	"os"
 	"path"
 	"time"
@@ -32,6 +33,7 @@ import (
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/remotefile"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
@@ -95,9 +97,48 @@ func (s *SLocalStorage) SyncStorageSize() error {
 	return err
 }
 
+func (s *SLocalStorage) GetAvailSizeMb() int {
+	sizeMb := s.SBaseStorage.GetAvailSizeMb()
+	kubeletConf := s.Manager.GetKubeletConfig()
+	if kubeletConf == nil {
+		return sizeMb
+	}
+
+	// available size should aware of kubelet hard eviction threshold
+	hardThresholds := kubeletConf.GetEvictionConfig().GetHard()
+	nodeFs := hardThresholds.GetNodeFsAvailable()
+	imageFs := hardThresholds.GetImageFsAvailable()
+	storageDev, err := kubelet.GetDirectoryMountDevice(s.GetPath())
+	if err != nil {
+		log.Errorf("Get directory %s mount device: %v", s.GetPath(), err)
+		return sizeMb
+	}
+
+	usablePercent := 1.0
+	if kubeletConf.HasDedicatedImageFs() {
+		if storageDev == kubeletConf.GetImageFsDevice() {
+			usablePercent = 1 - float64(imageFs.Value.Percentage)
+			log.Infof("Storage %s and kubelet imageFs %s share same device %s", s.GetPath(), kubeletConf.GetImageFs(), storageDev)
+		}
+	} else {
+		// nodeFs and imageFs use same device
+		if storageDev == kubeletConf.GetNodeFsDevice() {
+			maxPercent := math.Max(float64(nodeFs.Value.Percentage), float64(imageFs.Value.Percentage))
+			usablePercent = 1 - maxPercent
+			log.Infof("Storage %s and kubelet nodeFs share same device %s", s.GetPath(), storageDev)
+		}
+	}
+
+	sizeMb = int(float64(sizeMb) * usablePercent)
+	log.Infof("Storage %s sizeMb %d, usablePercent %f", s.GetPath(), sizeMb, usablePercent)
+
+	return sizeMb
+}
+
 func (s *SLocalStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
 	content := jsonutils.NewDict()
-	content.Set("name", jsonutils.NewString(s.GetName(s.GetComposedName)))
+	name := s.GetName(s.GetComposedName)
+	content.Set("name", jsonutils.NewString(name))
 	content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb())))
 	content.Set("actual_capacity_used", jsonutils.NewInt(int64(s.GetUsedSizeMb())))
 	content.Set("storage_type", jsonutils.NewString(s.StorageType()))
@@ -112,7 +153,7 @@ func (s *SLocalStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
 		res jsonutils.JSONObject
 	)
 
-	log.Infof("Sync storage info %s", s.StorageId)
+	log.Infof("Sync storage info %s/%s", s.StorageId, name)
 
 	if len(s.StorageId) > 0 {
 		res, err = modules.Storages.Put(

pkg/util/cgrouputils/cgrouputils.go

@@ -317,12 +317,12 @@ func (c *CGroupTask) init() bool {
 	if !CgroupIsMounted() {
 		if !fileutils2.Exists(cgroupsPath) {
 			if err := procutils.NewCommand("mkdir", "-p", cgroupsPath).Run(); err != nil {
-				log.Errorln(err)
+				log.Errorf("mkdir -p %s error: %v", cgroupsPath, err)
 			}
 		}
 		if err := procutils.NewCommand("mount", "-t", "tmpfs", "-o", "uid=0,gid=0,mode=0755",
 			"cgroup", cgroupsPath).Run(); err != nil {
-			log.Errorln(err)
+			log.Errorf("mount cgroups path %s, error: %v", cgroupsPath, err)
 			return false
 		}
 	}
@@ -350,7 +350,7 @@ func (c *CGroupTask) init() bool {
 				}
 				if err := procutils.NewCommand("mount", "-t", "cgroup", "-o",
 					module, module, moduleDir).Run(); err != nil {
-					log.Errorln(err)
+					log.Errorf("mount cgroup module %s to %s error: %v", module, moduleDir, err)
 					return false
 				}
 			}
@@ -358,7 +358,7 @@ func (c *CGroupTask) init() bool {
 	}
 
 	if err := scanner.Err(); err != nil {
-		log.Errorln(err)
+		log.Errorf("scan file %s error: %v", file.Name(), err)
 		return false
 	}
 

pkg/util/fileutils2/fileutils.go

@@ -143,7 +143,7 @@ func ChangeAllBlkdevsParams(params map[string]string) {
 	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
 		blockDevs, err := ioutil.ReadDir("/sys/block")
 		if err != nil {
-			log.Errorln(err)
+			log.Errorf("ReadDir /sys/block error: %s", err)
 			return
 		}
 		for _, b := range blockDevs {