From e0c13bb814b9dde3f81fc4749cb4a31c5b6f37ca Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Fri, 8 Aug 2025 20:54:56 +0800 Subject: [PATCH] feat(host): container cpufreq realtime simulate (#23047) --- pkg/hostman/guestman/guestman.go | 18 ++++ pkg/hostman/guestman/pod.go | 24 ++++- pkg/hostman/guestman/pod_helper.go | 101 ++++++++++++++++++ .../container_nvidia_gpu_metrics.go | 2 +- pkg/hostman/options/options.go | 2 + 5 files changed, 143 insertions(+), 4 deletions(-) diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index c20265d0e0..e878ce7d5a 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -121,6 +121,7 @@ type SGuestManager struct { containerRuntimeManager runtime.Runtime pleg pleg.PodLifecycleEventGenerator podCache runtime.Cache + cpufreqSimulateManager *SCpuFreqRealTimeSimulateManager } func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (*SGuestManager, error) { @@ -163,6 +164,16 @@ func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (* log.Fatalf("start containerd snapshot service: %s", err) } }() + if options.HostOptions.EnableRealtimeCpufreqSimulate { + cpufreqConfig := manager.host.GetContainerCpufreqSimulateConfig() + if cpufreqConfig != nil { + maxFreq, _ := cpufreqConfig.Int("scaling_max_freq") + minFreq, _ := cpufreqConfig.Int("scaling_min_freq") + interval := options.HostOptions.RealtimeCpufreqSimulateInterval + manager.cpufreqSimulateManager = newCpuFreqRealTimeSimulateManager(interval, maxFreq, minFreq) + } + } + } return manager, nil } @@ -383,6 +394,10 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) { log.Infof("[%s removed] enable dirty recovery feature at next bootstrap", m.disableDirtyRecoveryFilePath()) } }) + if m.cpufreqSimulateManager != nil { + go m.cpufreqSimulateManager.StartSetCpuFreqSimulate() + } + return m.dirtyServersChan, nil } @@ -1645,6 +1660,9 @@ func (m *SGuestManager) HotplugCpuMem(ctx context.Context, params interface{}) ( } func (m *SGuestManager) ExitGuestCleanup() { + if m.cpufreqSimulateManager != nil { + m.cpufreqSimulateManager.Stop() + } m.Servers.Range(func(k, v interface{}) bool { guest := v.(GuestRuntimeInstance) guest.ExitCleanup(false) diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 0445016c3e..dc22136356 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -864,7 +864,7 @@ func (s *sPodGuestInstance) getOtherPods() []*sPodGuestInstance { return otherPods } -func (s *sPodGuestInstance) getCgroupParent() string { +func PodCgroupParent() string { return "/cloudpods" } @@ -1004,7 +1004,7 @@ func (s *sPodGuestInstance) _startPod(ctx context.Context, userCred mcclient.Tok }, Annotations: nil, Linux: &runtimeapi.LinuxPodSandboxConfig{ - CgroupParent: s.getCgroupParent(), + CgroupParent: PodCgroupParent(), SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ NamespaceOptions: s.namespacesForPod(podInput), SelinuxOptions: nil, @@ -1696,7 +1696,7 @@ func (s *sPodGuestInstance) getContainerMounts(ctrId string, input *hostapi.Cont } func (s *sPodGuestInstance) getCGUtil() pod.CgroupUtil { - return pod.NewPodCgroupV1Util(s.getCgroupParent()) + return pod.NewPodCgroupV1Util(PodCgroupParent()) } func (s *sPodGuestInstance) setContainerCgroupDevicesAllow(ctrId string, allowStrs []string) error { @@ -2091,6 +2091,24 @@ func (s *sPodGuestInstance) findHostCpuPath(ctrId string, cpuIndex int) (int, er return s.getHostCPUMap().Get(ctrId, cpuIndex) } +func (s *sPodGuestInstance) simulateContainerSystemCpuSetScalingCurFreq(ctrId string, scalingCurFreq int64) error { + cpuDir := s.getContainerSystemCpusDir(ctrId) + cpuCnt := s.GetDesc().Cpu + for i := 0; i < int(cpuCnt); i++ { + cpufreqPolicyCurFreqFile := path.Join(cpuDir, "cpufreq", fmt.Sprintf("policy%d", i), "scaling_cur_freq") + cpufreqPolicySetSpeedFile := path.Join(cpuDir, "cpufreq", fmt.Sprintf("policy%d", i), "scaling_setspeed") + scalingCurFreqStr := fmt.Sprintf("%d\n", scalingCurFreq) + if err := fileutils2.FilePutContents(cpufreqPolicyCurFreqFile, scalingCurFreqStr, false); err != nil { + return errors.Wrapf(err, "failed write %s", cpufreqPolicyCurFreqFile) + } + if err := fileutils2.FilePutContents(cpufreqPolicySetSpeedFile, scalingCurFreqStr, false); err != nil { + return errors.Wrapf(err, "failed write %s", cpufreqPolicySetSpeedFile) + } + } + + return nil +} + func (s *sPodGuestInstance) simulateContainerSystemCpu(ctx context.Context, ctrId string) ([]*runtimeapi.Mount, error) { cpuDir := s.getContainerSystemCpusDir(ctrId) cpuCnt := s.GetDesc().Cpu diff --git a/pkg/hostman/guestman/pod_helper.go b/pkg/hostman/guestman/pod_helper.go index 46be008265..98b51dccf7 100644 --- a/pkg/hostman/guestman/pod_helper.go +++ b/pkg/hostman/guestman/pod_helper.go @@ -17,7 +17,10 @@ package guestman import ( "context" "fmt" + "path" + "strconv" "strings" + "time" "github.com/shirou/gopsutil/v3/disk" @@ -30,6 +33,8 @@ import ( "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/cgrouputils" + "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/pod/image" "yunion.io/x/onecloud/pkg/util/pod/nerdctl" ) @@ -244,3 +249,99 @@ func GetPodStatusByContainerStatus(status string, cStatus string, isPrimary bool } return status } + +type SCpuFreqRealTimeSimulateManager struct { + lastTimeCpuUsage map[string]int64 + lastTime *int64 + + interval int + stop chan struct{} + + cpufreqMax int64 + cpufreqMin int64 +} + +func newCpuFreqRealTimeSimulateManager(intervalSecond int, cpufreqMax, cpufreqMin int64) *SCpuFreqRealTimeSimulateManager { + return &SCpuFreqRealTimeSimulateManager{ + interval: intervalSecond, + stop: make(chan struct{}), + cpufreqMax: cpufreqMax, + cpufreqMin: cpufreqMin, + } +} + +func (m *SCpuFreqRealTimeSimulateManager) StartSetCpuFreqSimulate() { + log.Infof("StartSetCpuFreqSimulate") + ticker := time.NewTicker(time.Duration(m.interval) * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + m.startSetCpuFreqSimulate() + case <-m.stop: + return + } + } +} + +func (m *SCpuFreqRealTimeSimulateManager) Stop() { + close(m.stop) +} + +func (m *SCpuFreqRealTimeSimulateManager) startSetCpuFreqSimulate() { + newCpuUsage := map[string]int64{} + startTime := time.Now().UnixNano() + guestManager.Servers.Range(func(k, v interface{}) bool { + pod, ok := v.(*sPodGuestInstance) + if !ok { + log.Errorf("is not pod instance") + return false + } + cgroupRoot := path.Join(cgrouputils.GetSubModulePath("cpuacct"), PodCgroupParent()) + criIds := pod.GetPodContainerCriIds() + for i := range criIds { + ctr, err := pod.GetContainerByCRIId(criIds[i]) + if err != nil { + log.Errorf("failed get %s ctrid by criId %s", pod.GetName(), criIds[i]) + continue + } + cpuDir := pod.getContainerSystemCpusDir(ctr.Id) + if !fileutils2.Exists(cpuDir) { + log.Errorf("%s %s has no cpuDir", pod.GetName(), criIds[i]) + continue + } + + usagePath := path.Join(cgroupRoot, criIds[i], "cpuacct.usage") + criCpuUsageStr, err := fileutils2.FileGetContents(usagePath) + if err != nil { + continue + } + criCpuUsageStr = strings.TrimSpace(criCpuUsageStr) + criCpuUsage, err := strconv.ParseInt(criCpuUsageStr, 10, 0) + if err != nil { + log.Errorf("failed parse %s %s: %s", usagePath, criCpuUsageStr, err) + continue + } + + newCpuUsage[criIds[i]] = criCpuUsage + if m.lastTime != nil { + if lastTimeUsage, ok := m.lastTimeCpuUsage[criIds[i]]; ok { + timeDiff := float64(startTime - *m.lastTime) + usageDiff := float64(criCpuUsage - lastTimeUsage) + cpuUsagePercent := (usageDiff / timeDiff) * 100 + freqRange := float64(m.cpufreqMax - m.cpufreqMin) + estimatedFreq := m.cpufreqMin + int64(freqRange*cpuUsagePercent/100) + + err := pod.simulateContainerSystemCpuSetScalingCurFreq(ctr.Id, estimatedFreq) + if err != nil { + log.Errorf("failed set %s(%s) simulate cpufreq: %s", pod.GetId(), criIds[i], err) + } + } + } + } + return true + }) + m.lastTime = &startTime + m.lastTimeCpuUsage = newCpuUsage +} diff --git a/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go b/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go index daf0d0fcce..c879c468ef 100644 --- a/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go +++ b/pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go @@ -147,7 +147,7 @@ func parseNvidiaGpuProcessMetrics(gpuMetricsStr string) []NvidiaGpuProcessMetric func (s *SGuestMonitorCollector) collectGpuPodsProcesses() map[string]map[string]struct{} { podProcIds := map[string]map[string]struct{}{} guestmanager := guestman.GetGuestManager() - cgroupRoot := path.Join(cgrouputils.GetSubModulePath("cpuset"), "cloudpods") + cgroupRoot := path.Join(cgrouputils.GetSubModulePath("cpuset"), guestman.PodCgroupParent()) guestmanager.Servers.Range(func(k, v interface{}) bool { pod, ok := v.(guestman.PodInstance) if !ok { diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index ca7f04f055..dfbee7270e 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -240,6 +240,8 @@ type SHostOptions struct { ContainerDeviceConfigFile string `help:"container device configuration file path"` LxcfsPath string `help:"lxcfs directory path" default:"/var/lib/lxcfs"` ContainerSystemCpufreqSimulateConfigFile string `help:"container system cpu simulate config file path" default:"/etc/yunion/container_cpufreq_simulate.conf"` + EnableRealtimeCpufreqSimulate bool `help:"realtime cpufreq simulate" default:"true"` + RealtimeCpufreqSimulateInterval int `help:"realtime cpufreq simulate interval(second)" default:"2"` EnableCudaMPS bool `help:"enable cuda mps" default:"false"` CudaMPSPipeDirectory string `help:"cuda mps pipe dir" default:"/tmp/nvidia-mps/pipe"`