diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index 9e9d491a88..746c12304e 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -81,12 +81,12 @@ func (host *SHostService) RunService() { hostInstance := hostinfo.Instance() if err := hostInstance.Init(); err != nil { - log.Fatalf(err.Error()) + log.Fatalf("Host instance init error: %v", err) } deployclient.Init(options.HostOptions.DeployServerSocketPath) if err := storageman.Init(hostInstance); err != nil { - log.Fatalf(err.Error()) + log.Fatalf("Storage manager init error: %v", err) } var guestChan chan struct{} @@ -95,7 +95,7 @@ func (host *SHostService) RunService() { log.Infof("Auth complete!!") if err := host.initEtcdConfig(); err != nil { - log.Fatalln(err) + log.Fatalln("Init etcd config: %v", err) } hostInstance.StartRegister(2, func() { diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 27f0d72bec..1199461833 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -43,6 +43,7 @@ import ( "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts" "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" @@ -72,6 +73,8 @@ type SHostInfo struct { Mem *SMemory sysinfo *SSysInfo + kubeletConfig kubelet.KubeletConfig + isInit bool enableHugePages bool onHostDown string @@ -158,7 +161,7 @@ func (h *SHostInfo) IsHugepagesEnabled() bool { */ func (h *SHostInfo) Init() error { if err := h.prepareEnv(); err != nil { - return err + return errors.Wrap(err, "Prepare environment") } log.Infof("Start detectHostInfo") @@ -317,7 +320,7 @@ func (h *SHostInfo) parseConfig() error { func (h *SHostInfo) prepareEnv() error { if err := h.fixPathEnv(); err != nil { - return err + return errors.Wrap(err, "Fix path environment") } if options.HostOptions.ReportInterval > 300 { return fmt.Errorf("Option report_interval must no longer than 5 min") @@ -330,7 +333,7 @@ func (h *SHostInfo) prepareEnv() error { _, err = procutils.NewCommand("ethtool", "-h").Output() if err != nil { - return fmt.Errorf("Ethtool not installed") + return errors.Wrap(err, "Execute 'ethtool -h'") } ioParams := make(map[string]string, 0) @@ -346,11 +349,11 @@ func (h *SHostInfo) prepareEnv() error { fileutils2.ChangeAllBlkdevsParams(ioParams) _, err = procutils.NewRemoteCommandAsFarAsPossible("modprobe", "tun").Output() if err != nil { - return fmt.Errorf("Failed to activate tun/tap device") + return errors.Wrap(err, "Failed to activate tun/tap device") } output, err = procutils.NewRemoteCommandAsFarAsPossible("modprobe", "vhost_net").Output() if err != nil { - log.Errorf("modprobe error: %s", output) + log.Warningf("modprobe vhost_net error: %s", output) } if !options.HostOptions.DisableSetCgroup { if !cgrouputils.Init() { @@ -495,7 +498,16 @@ func (h *SHostInfo) GetMemory() (int, error) { if options.HostOptions.HugepagesOption == "native" { return h.Mem.GetHugepageTotal() } - return h.Mem.Total, nil // - options.reserved_memory + total := h.Mem.Total + if h.kubeletConfig != nil { + memThreshold := h.kubeletConfig.GetEvictionConfig().GetHard().GetMemoryAvailable() + memBytes, _ := memThreshold.Value.Quantity.AsInt64() + memMb := int(memBytes / 1024 / 1024) + subMem := total - memMb + log.Infof("Get total memory %d, kubelet memory threshold subtracted: (%d - %d)", subMem, total, memMb) + total = subMem + } + return total, nil // - options.reserved_memory } func (h *SHostInfo) getCurrentHugepageNr() (int64, error) { @@ -1394,11 +1406,11 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) { params.Set("is_root_partiton", jsonutils.JSONTrue) _, err := modules.Hoststorages.Update(h.GetSession(), h.HostId, storageId, nil, params) if err != nil { - h.onFail(err) + h.onFail(errors.Wrapf(err, "Update host storage %s with params %s", storageId, params)) } } if err := storage.SetStorageInfo(storageId, storageName, storageConf); err != nil { - h.onFail(err) + h.onFail(errors.Wrapf(err, "Set storage info %s/%s/%s", storageId, storageName, storageConf)) } } else { // XXX hack: storage type baremetal is a converted host，reserve storage @@ -1419,11 +1431,11 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) { func (h *SHostInfo) uploadStorageInfo() { for _, s := range storageman.GetManager().Storages { if err := s.SetStorageInfo(s.GetId(), s.GetStorageName(), s.GetStorageConf()); err != nil { - h.onFail(err) + h.onFail(errors.Wrapf(err, "Upload storage %s info with config %s", s.GetStorageName(), s.GetStorageConf())) } res, err := s.SyncStorageInfo() if err != nil { - h.onFail(err) + h.onFail(errors.Wrapf(err, "Sync storage %s info", s.GetStorageName())) } else { h.onSyncStorageInfoSucc(s, res) } @@ -1750,6 +1762,10 @@ func (h *SHostInfo) IsX8664() bool { return h.GetCpuArchitecture() == apis.OS_ARCH_X86_64 } +func (h *SHostInfo) GetKubeletConfig() kubelet.KubeletConfig { + return h.kubeletConfig +} + func NewHostInfo() (*SHostInfo, error) { var res = new(SHostInfo) res.sysinfo = &SSysInfo{} @@ -1779,6 +1795,17 @@ func NewHostInfo() (*SHostInfo, error) { res.IsRegistered = make(chan struct{}) res.SysError = make(map[string]string) res.SysWarning = make(map[string]string) + + if !options.HostOptions.DisableProbeKubelet { + kubeletDir := options.HostOptions.KubeletRunDirectory + kubeletConfig, err := kubelet.NewKubeletConfigByDirectory(kubeletDir) + if err != nil { + return nil, errors.Wrapf(err, "New kubelet config by dir: %s", kubeletDir) + } + res.kubeletConfig = kubeletConfig + log.Infof("Get kubelet container image Fs: %s, eviction config: %s", res.kubeletConfig.GetImageFs(), res.kubeletConfig.GetEvictionConfig()) + } + return res, nil } @@ -1789,7 +1816,7 @@ func Instance() *SHostInfo { var err error hostInfo, err = NewHostInfo() if err != nil { - log.Fatalln(err) + log.Fatalf("NewHostInfo: %s", err) } } return hostInfo diff --git a/pkg/hostman/hostutils/hostutils.go b/pkg/hostman/hostutils/hostutils.go index 32fcef7513..336027259e 100644 --- a/pkg/hostman/hostutils/hostutils.go +++ b/pkg/hostman/hostutils/hostutils.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/workmanager" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" + "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/httperrors" @@ -54,6 +55,8 @@ type IHost interface { GetIsolatedDeviceManager() *isolated_device.IsolatedDeviceManager SyncRootPartitionUsedCapacity() error + + GetKubeletConfig() kubelet.KubeletConfig } func GetComputeSession(ctx context.Context) *mcclient.ClientSession { diff --git a/pkg/hostman/hostutils/kubelet/doc.go b/pkg/hostman/hostutils/kubelet/doc.go new file mode 100644 index 0000000000..199eebb7c8 --- /dev/null +++ b/pkg/hostman/hostutils/kubelet/doc.go @@ -0,0 +1 @@ +package kubelet // import "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" diff --git a/pkg/hostman/hostutils/kubelet/docker.go b/pkg/hostman/hostutils/kubelet/docker.go new file mode 100644 index 0000000000..fa3b272f52 --- /dev/null +++ b/pkg/hostman/hostutils/kubelet/docker.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubelet + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/util/procutils" +) + +type DockerInfo struct { + ID string `json:"ID"` + Driver string `json:"Driver"` + DockerRootDir string `json:"DockerRootDir"` +} + +func GetDockerInfoByRemote() (*DockerInfo, error) { + content, err := procutils.NewRemoteCommandAsFarAsPossible("docker", "info", "--format", "{{json .}}").Output() + if err != nil { + return nil, errors.Wrap(err, "Run command 'docker info'") + } + + obj, err := jsonutils.Parse(content) + if err != nil { + return nil, errors.Wrap(err, "Parse docker info to json") + } + + info := new(DockerInfo) + if err := obj.Unmarshal(info); err != nil { + return nil, errors.Wrap(err, "Unmarshal docker info") + } + + return info, nil +} diff --git a/pkg/hostman/hostutils/kubelet/eviction/doc.go b/pkg/hostman/hostutils/kubelet/eviction/doc.go new file mode 100644 index 0000000000..016f952887 --- /dev/null +++ b/pkg/hostman/hostutils/kubelet/eviction/doc.go @@ -0,0 +1 @@ +package eviction // import "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet/eviction" diff --git a/pkg/hostman/hostutils/kubelet/eviction/eviction.go b/pkg/hostman/hostutils/kubelet/eviction/eviction.go new file mode 100644 index 0000000000..082392a0f3 --- /dev/null +++ b/pkg/hostman/hostutils/kubelet/eviction/eviction.go @@ -0,0 +1,264 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eviction + +import ( + "encoding/json" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +// Signal defines a signal that can trigger eviction of pods on a node. +type Signal string + +const ( + // SignalMemoryAvailable is memory available (i.e. capacity - workingSet), in bytes. + SignalMemoryAvailable Signal = "memory.available" + // SignalNodeFsAvailable is amount of storage available on filesystem that kubelet uses for volumes, daemon logs, etc. + SignalNodeFsAvailable Signal = "nodefs.available" + // SignalNodeFsInodesFree is amount of storage available on filesystem that container runtime uses for storing images and container writable layers. + SignalNodeFsInodesFree Signal = "nodefs.inodesFree" + // SignalImageFsAvailable is amount of storage available on filesystem that container runtime uses for storing images and container writable layers. + SignalImageFsAvailable Signal = "imagefs.available" + // SignalImageFsInodesFree is amount of inodes available on filesystem that container runtime uses for storing images and container writable layers. + SignalImageFsInodesFree Signal = "imagefs.inodesFree" + // SignalAllocatableMemoryAvailable is amount of memory available for pod allocation (i.e. allocatable - workingSet (of pods), in bytes) + // SignalAllocatableMemoryAvailable Signal = "allocatableMemory.available" + // SignalPIDAvailable is amount of PID available for pod allocation + SignalPIDAvailable Signal = "pid.available" +) + +var ( + // DefaultEvictionHard includes default options for kubelet hard eviction + // ref: https://github.com/kubernetes/kubernetes/blob/ec39cc2eafffa51b3267e3bd64fbd2598c0db94d/pkg/kubelet/apis/config/v1beta1/defaults_linux.go#L21 + DefaultEvictionHard = map[string]string{ + string(SignalMemoryAvailable): "100Mi", + string(SignalNodeFsAvailable): "10%", + string(SignalNodeFsInodesFree): "5%", + string(SignalImageFsAvailable): "15%", + } +) + +type ThresholdMap map[Signal]*Threshold + +func (m ThresholdMap) GetMemoryAvailable() *Threshold { + return m[SignalMemoryAvailable] +} + +func (m ThresholdMap) GetNodeFsAvailable() *Threshold { + return m[SignalNodeFsAvailable] +} + +func (m ThresholdMap) GetNodeFsInodesFree() *Threshold { + return m[SignalNodeFsInodesFree] +} + +func (m ThresholdMap) GetImageFsAvailable() *Threshold { + return m[SignalImageFsAvailable] +} + +// Config holds information about how eviction is configured. +type Config interface { + GetHard() ThresholdMap + String() string +} + +// config implements Config interface +type config struct { + // hard holds configuration of hardThresholds + hard ThresholdMap +} + +type configContent struct { + // Map of signal names to quantities that defines hard eviction thresholds. For example: {"memory.available": "300Mi"} + EvictionHard map[string]string `json:"evictionHard"` + // Map of signal names to quantities that defines soft eviction thresholds. For example: {"memory.available": "300Mi"} + // EvictionSoft map[string]string `json:"evictionSoft"` +} + +func NewConfig(yamlContent []byte) (Config, error) { + obj, err := jsonutils.ParseYAML(string(yamlContent)) + if err != nil { + return nil, errors.Wrapf(err, "Parse yaml content %q", yamlContent) + } + + content := new(configContent) + if err := obj.Unmarshal(content); err != nil { + return nil, errors.Wrap(err, "Unmarshal eviction content") + } + + hardThresholds, err := parseThresholdConfig(content.EvictionHard) + if err != nil { + return nil, errors.Wrap(err, "Parse hard thresholds") + } + + return &config{ + hard: hardThresholds, + }, nil +} + +func (c *config) GetHard() ThresholdMap { + return c.hard +} + +func (c *config) String() string { + out := map[string]interface{}{ + "evictionHard": c.hard, + } + + bytes, err := json.Marshal(out) + if err != nil { + log.Errorf("Marshal eviction config error: %v", err) + } + return string(bytes) +} + +func parseThresholdConfig(evictionHard map[string]string) (map[Signal]*Threshold, error) { + results := map[Signal]*Threshold{} + hardThresholds, err := parseHardThresholdStatements(evictionHard) + if err != nil { + return nil, errors.Wrap(err, "Parse hard threshold") + } + for _, r := range hardThresholds { + results[r.Signal] = r + } + return results, nil +} + +func parseHardThresholdStatements(statements map[string]string) ([]*Threshold, error) { + results := []*Threshold{} + + for _, signal := range []Signal{ + SignalMemoryAvailable, + SignalNodeFsAvailable, + SignalNodeFsInodesFree, + SignalImageFsAvailable, + SignalImageFsInodesFree, + SignalPIDAvailable, + } { + val, ok := statements[string(signal)] + if !ok { + // try get from default setting + val, ok = DefaultEvictionHard[string(signal)] + if !ok { + continue + } + } + result, err := parseThresholdStatement(signal, val) + if err != nil { + return nil, errors.Wrapf(err, "Parse signal %q with val %q", signal, val) + } + if result == nil { + continue + } + results = append(results, result) + } + return results, nil +} + +func parseThresholdStatement(signal Signal, val string) (*Threshold, error) { + operator, ok := OpForSignal[signal] + if !ok { + return nil, errors.Errorf("Unsupported signal %q", signal) + } + if strings.HasSuffix(val, "%") { + // ignore 0% and 100% + if val == "0%" || val == "100%" { + return nil, nil + } + percentage, err := parsePercentage(val) + if err != nil { + return nil, errors.Wrapf(err, "Parse val %q to percentage", val) + } + if percentage < 0 { + return nil, errors.Errorf("Eviction percentage threshold %q must be >= 0%%: %q", signal, val) + } + if percentage > 100 { + return nil, errors.Errorf("Eviction percentage threshold %q must be <= 100%%: %q", signal, val) + } + return &Threshold{ + Signal: signal, + Operator: operator, + Value: ThresholdValue{ + Percentage: percentage, + }, + }, nil + } + quantity, err := resource.ParseQuantity(val) + if err != nil { + return nil, err + } + if quantity.Sign() < 0 || quantity.IsZero() { + return nil, errors.Errorf("Eviction threshold %q must be positive: %s", signal, &quantity) + } + return &Threshold{ + Signal: signal, + Operator: operator, + Value: ThresholdValue{ + Quantity: &quantity, + }, + }, nil +} + +func parsePercentage(input string) (float32, error) { + val, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32) + if err != nil { + return 0, err + } + return float32(val) / 100, nil +} + +// ThresholdOperator is the operator used to express a Threshold. +type ThresholdOperator string + +const ( + // OpLessThan is the operator that expresses a less than operator. + OpLessThan ThresholdOperator = "LessThan" +) + +// OpForSignal maps Signals to ThresholdOperators. +// Today, the only supported operator is "LessThan". +var OpForSignal = map[Signal]ThresholdOperator{ + SignalMemoryAvailable: OpLessThan, + SignalNodeFsAvailable: OpLessThan, + SignalNodeFsInodesFree: OpLessThan, + SignalImageFsAvailable: OpLessThan, + SignalImageFsInodesFree: OpLessThan, + SignalPIDAvailable: OpLessThan, +} + +// Threshold defines a metric for when eviction should occur. +type Threshold struct { + // Signal defines the entity that was measured. + Signal Signal + // Operator represents a relationship of a signal to a value. + Operator ThresholdOperator + // Value is the threshold the resource is evaluated against. + Value ThresholdValue +} + +// ThresholdValue is a value holder that abstracts literal versus percentage based quantity +type ThresholdValue struct { + // Quantity is a quantity associated with the signal + Quantity *resource.Quantity + // Percentage represents the usage percentage over the total resource + Percentage float32 +} diff --git a/pkg/hostman/hostutils/kubelet/eviction/eviction_test.go b/pkg/hostman/hostutils/kubelet/eviction/eviction_test.go new file mode 100644 index 0000000000..2b10a7189d --- /dev/null +++ b/pkg/hostman/hostutils/kubelet/eviction/eviction_test.go @@ -0,0 +1,91 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eviction + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewConfig(t *testing.T) { + type tCase struct { + name string + input string + assertFunc func(*testing.T, Config) error + } + + cases := []tCase{ + { + name: "Should has default config with empty input", + input: ``, + assertFunc: func(t *testing.T, config Config) error { + memBytes, _ := config.GetHard().GetMemoryAvailable().Value.Quantity.AsInt64() + assert.Equal(t, int64(1024*1024*100), memBytes) + assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsAvailable().Value.Percentage) + assert.Equal(t, float32(0.05), config.GetHard().GetNodeFsInodesFree().Value.Percentage) + assert.Equal(t, float32(0.15), config.GetHard().GetImageFsAvailable().Value.Percentage) + return nil + }, + }, + { + name: "With all config", + input: ` +evictionHard: + imagefs.available: 25% + memory.available: 1024Mi + nodefs.available: 15% + nodefs.inodesFree: 10%`, + assertFunc: func(t *testing.T, config Config) error { + memBytes, _ := config.GetHard().GetMemoryAvailable().Value.Quantity.AsInt64() + assert.Equal(t, int64(1024*1024*1024), memBytes) + assert.Equal(t, float32(0.25), config.GetHard().GetImageFsAvailable().Value.Percentage) + assert.Equal(t, float32(0.15), config.GetHard().GetNodeFsAvailable().Value.Percentage) + assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsInodesFree().Value.Percentage) + return nil + }, + }, + { + name: "Mixed config with default", + input: ` +evictionHard: + imagefs.available: 5% + nodefs.inodesFree: 10%`, + assertFunc: func(t *testing.T, config Config) error { + memBytes, _ := config.GetHard().GetMemoryAvailable().Value.Quantity.AsInt64() + assert.Equal(t, int64(1024*1024*100), memBytes) + assert.Equal(t, float32(0.05), config.GetHard().GetImageFsAvailable().Value.Percentage) + assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsAvailable().Value.Percentage) + assert.Equal(t, float32(0.1), config.GetHard().GetNodeFsInodesFree().Value.Percentage) + return nil + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(st *testing.T) { + if config, err := NewConfig([]byte(tc.input)); err != nil { + st.Errorf("[%s] NewConfig error: %v", tc.name, err) + } else { + str, _ := json.MarshalIndent(config, "", " ") + st.Logf("%s", str) + if err := tc.assertFunc(st, config); err != nil { + st.Errorf("[%s] assert error: %v", tc.name, err) + } + } + }) + } +} diff --git a/pkg/hostman/hostutils/kubelet/kubelet.go b/pkg/hostman/hostutils/kubelet/kubelet.go new file mode 100644 index 0000000000..5c2c7cef89 --- /dev/null +++ b/pkg/hostman/hostutils/kubelet/kubelet.go @@ -0,0 +1,124 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubelet + +import ( + "path" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet/eviction" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +const ( + KubeletConfigurationFileName = "config.yaml" +) + +// KubeletConfig is a interface abstracts manipulation of kubelet run directory. +type KubeletConfig interface { + HasDedicatedImageFs() bool + GetNodeFsDevice() string + GetImageFsDevice() string + GetImageFs() string + GetEvictionConfig() eviction.Config +} + +// kubeletConfig implements KubeletRunDirectory interface. +type kubeletConfig struct { + config jsonutils.JSONObject + dockerInfo *DockerInfo + evictionConfig eviction.Config + nodeFsDevice string + imageFsDevice string +} + +func NewKubeletConfigByDirectory(dir string) (KubeletConfig, error) { + configFile := path.Join(dir, KubeletConfigurationFileName) + + content, err := procutils.NewRemoteCommandAsFarAsPossible("cat", configFile).Output() + if err != nil { + return nil, errors.Wrapf(err, "Read config file %s", configFile) + } + + dockerInfo, err := GetDockerInfoByRemote() + if err != nil { + return nil, errors.Wrap(err, "Get docker info") + } + + return newKubeletConfig(content, dockerInfo) +} + +func newKubeletConfig(yamlConfig []byte, dockerInfo *DockerInfo) (KubeletConfig, error) { + obj, err := jsonutils.ParseYAML(string(yamlConfig)) + if err != nil { + return nil, errors.Wrapf(err, "Parse yaml content %s", yamlConfig) + } + + evictionConfig, err := eviction.NewConfig(yamlConfig) + if err != nil { + return nil, errors.Wrap(err, "New eviction config") + } + + imageFsDev, err := GetDirectoryMountDevice(dockerInfo.DockerRootDir) + if err != nil { + return nil, errors.Wrap(err, "Find docker root directory device") + } + + nodeFsDev, err := GetDirectoryMountDevice("/") + if err != nil { + return nil, errors.Wrap(err, "Find node FS directory device") + } + + k := &kubeletConfig{ + config: obj, + dockerInfo: dockerInfo, + evictionConfig: evictionConfig, + nodeFsDevice: nodeFsDev, + imageFsDevice: imageFsDev, + } + + return k, nil +} + +func GetDirectoryMountDevice(dirPath string) (string, error) { + content, err := procutils.NewRemoteCommandAsFarAsPossible("findmnt", "-n", "-o", "SOURCE", "--target", dirPath).Output() + if err != nil { + return "", errors.Wrapf(err, "Find directory %q mount source device", dirPath) + } + return strings.TrimSpace(string(content)), nil +} + +func (k *kubeletConfig) GetNodeFsDevice() string { + return k.nodeFsDevice +} + +func (k *kubeletConfig) GetImageFsDevice() string { + return k.imageFsDevice +} + +func (k *kubeletConfig) HasDedicatedImageFs() bool { + return k.imageFsDevice != k.nodeFsDevice +} + +func (k *kubeletConfig) GetImageFs() string { + return k.dockerInfo.DockerRootDir +} + +func (k *kubeletConfig) GetEvictionConfig() eviction.Config { + return k.evictionConfig +} diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index dffb05035e..8f5331a2dc 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -140,6 +140,9 @@ type SHostOptions struct { SyncStorageInfoDurationSecond int `help:"sync storage size duration, unit is second" default:"60"` StartHostIgnoreSysError bool `help:"start host agent ignore sys error" default:"false"` + + DisableProbeKubelet bool `help:"Disable probe kubelet config" default:"false"` + KubeletRunDirectory string `help:"Kubelet config file path" default:"/var/lib/kubelet"` } var ( diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index 8282ce34c9..db829b1132 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -29,6 +29,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman/storageutils" "yunion.io/x/onecloud/pkg/mcclient" @@ -73,7 +74,7 @@ func NewStorageManager(host hostutils.IHost) (*SStorageManager, error) { allFull = false } } else { - log.Errorf("storage %s not accessible", s.Path) + log.Errorf("storage %s not accessible error: %v", s.Path, err) } } @@ -122,6 +123,10 @@ func (s *SStorageManager) GetMediumType() string { return s.host.GetMediumType() } +func (s *SStorageManager) GetKubeletConfig() kubelet.KubeletConfig { + return s.host.GetKubeletConfig() +} + func (s *SStorageManager) getLeasedUsedLocalStorage(cacheDir string, limit int) (string, error) { var ( maxFree int diff --git a/pkg/hostman/storageman/storage_local.go b/pkg/hostman/storageman/storage_local.go index 4620d710fb..e12fd0aea2 100644 --- a/pkg/hostman/storageman/storage_local.go +++ b/pkg/hostman/storageman/storage_local.go @@ -17,6 +17,7 @@ package storageman import ( "context" "fmt" + "math" "os" "path" "time" @@ -32,6 +33,7 @@ import ( deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" "yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient" "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/hostutils/kubelet" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman/remotefile" "yunion.io/x/onecloud/pkg/mcclient/modules" @@ -95,9 +97,48 @@ func (s *SLocalStorage) SyncStorageSize() error { return err } +func (s *SLocalStorage) GetAvailSizeMb() int { + sizeMb := s.SBaseStorage.GetAvailSizeMb() + kubeletConf := s.Manager.GetKubeletConfig() + if kubeletConf == nil { + return sizeMb + } + + // available size should aware of kubelet hard eviction threshold + hardThresholds := kubeletConf.GetEvictionConfig().GetHard() + nodeFs := hardThresholds.GetNodeFsAvailable() + imageFs := hardThresholds.GetImageFsAvailable() + storageDev, err := kubelet.GetDirectoryMountDevice(s.GetPath()) + if err != nil { + log.Errorf("Get directory %s mount device: %v", s.GetPath(), err) + return sizeMb + } + + usablePercent := 1.0 + if kubeletConf.HasDedicatedImageFs() { + if storageDev == kubeletConf.GetImageFsDevice() { + usablePercent = 1 - float64(imageFs.Value.Percentage) + log.Infof("Storage %s and kubelet imageFs %s share same device %s", s.GetPath(), kubeletConf.GetImageFs(), storageDev) + } + } else { + // nodeFs and imageFs use same device + if storageDev == kubeletConf.GetNodeFsDevice() { + maxPercent := math.Max(float64(nodeFs.Value.Percentage), float64(imageFs.Value.Percentage)) + usablePercent = 1 - maxPercent + log.Infof("Storage %s and kubelet nodeFs share same device %s", s.GetPath(), storageDev) + } + } + + sizeMb = int(float64(sizeMb) * usablePercent) + log.Infof("Storage %s sizeMb %d, usablePercent %f", s.GetPath(), sizeMb, usablePercent) + + return sizeMb +} + func (s *SLocalStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { content := jsonutils.NewDict() - content.Set("name", jsonutils.NewString(s.GetName(s.GetComposedName))) + name := s.GetName(s.GetComposedName) + content.Set("name", jsonutils.NewString(name)) content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb()))) content.Set("actual_capacity_used", jsonutils.NewInt(int64(s.GetUsedSizeMb()))) content.Set("storage_type", jsonutils.NewString(s.StorageType())) @@ -112,7 +153,7 @@ func (s *SLocalStorage) SyncStorageInfo() (jsonutils.JSONObject, error) { res jsonutils.JSONObject ) - log.Infof("Sync storage info %s", s.StorageId) + log.Infof("Sync storage info %s/%s", s.StorageId, name) if len(s.StorageId) > 0 { res, err = modules.Storages.Put( diff --git a/pkg/util/cgrouputils/cgrouputils.go b/pkg/util/cgrouputils/cgrouputils.go index c4d28644c4..b338cdc51c 100644 --- a/pkg/util/cgrouputils/cgrouputils.go +++ b/pkg/util/cgrouputils/cgrouputils.go @@ -317,12 +317,12 @@ func (c *CGroupTask) init() bool { if !CgroupIsMounted() { if !fileutils2.Exists(cgroupsPath) { if err := procutils.NewCommand("mkdir", "-p", cgroupsPath).Run(); err != nil { - log.Errorln(err) + log.Errorf("mkdir -p %s error: %v", cgroupsPath, err) } } if err := procutils.NewCommand("mount", "-t", "tmpfs", "-o", "uid=0,gid=0,mode=0755", "cgroup", cgroupsPath).Run(); err != nil { - log.Errorln(err) + log.Errorf("mount cgroups path %s, error: %v", cgroupsPath, err) return false } } @@ -350,7 +350,7 @@ func (c *CGroupTask) init() bool { } if err := procutils.NewCommand("mount", "-t", "cgroup", "-o", module, module, moduleDir).Run(); err != nil { - log.Errorln(err) + log.Errorf("mount cgroup module %s to %s error: %v", module, moduleDir, err) return false } } @@ -358,7 +358,7 @@ func (c *CGroupTask) init() bool { } if err := scanner.Err(); err != nil { - log.Errorln(err) + log.Errorf("scan file %s error: %v", file.Name(), err) return false } diff --git a/pkg/util/fileutils2/fileutils.go b/pkg/util/fileutils2/fileutils.go index e2078ea847..6fedbe1944 100644 --- a/pkg/util/fileutils2/fileutils.go +++ b/pkg/util/fileutils2/fileutils.go @@ -143,7 +143,7 @@ func ChangeAllBlkdevsParams(params map[string]string) { if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) { blockDevs, err := ioutil.ReadDir("/sys/block") if err != nil { - log.Errorln(err) + log.Errorf("ReadDir /sys/block error: %s", err) return } for _, b := range blockDevs {