feat(host): container cpufreq realtime simulate (#23047)

pkg/hostman/guestman/guestman.go

@@ -121,6 +121,7 @@ type SGuestManager struct {
 	containerRuntimeManager    runtime.Runtime
 	pleg                       pleg.PodLifecycleEventGenerator
 	podCache                   runtime.Cache
+	cpufreqSimulateManager     *SCpuFreqRealTimeSimulateManager
 }
 
 func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (*SGuestManager, error) {
@@ -163,6 +164,16 @@ func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (*
 				log.Fatalf("start containerd snapshot service: %s", err)
 			}
 		}()
+		if options.HostOptions.EnableRealtimeCpufreqSimulate {
+			cpufreqConfig := manager.host.GetContainerCpufreqSimulateConfig()
+			if cpufreqConfig != nil {
+				maxFreq, _ := cpufreqConfig.Int("scaling_max_freq")
+				minFreq, _ := cpufreqConfig.Int("scaling_min_freq")
+				interval := options.HostOptions.RealtimeCpufreqSimulateInterval
+				manager.cpufreqSimulateManager = newCpuFreqRealTimeSimulateManager(interval, maxFreq, minFreq)
+			}
+		}
+
 	}
 	return manager, nil
 }
@@ -383,6 +394,10 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) {
 			log.Infof("[%s removed] enable dirty recovery feature at next bootstrap", m.disableDirtyRecoveryFilePath())
 		}
 	})
+	if m.cpufreqSimulateManager != nil {
+		go m.cpufreqSimulateManager.StartSetCpuFreqSimulate()
+	}
+
 	return m.dirtyServersChan, nil
 }
 
@@ -1645,6 +1660,9 @@ func (m *SGuestManager) HotplugCpuMem(ctx context.Context, params interface{}) (
 }
 
 func (m *SGuestManager) ExitGuestCleanup() {
+	if m.cpufreqSimulateManager != nil {
+		m.cpufreqSimulateManager.Stop()
+	}
 	m.Servers.Range(func(k, v interface{}) bool {
 		guest := v.(GuestRuntimeInstance)
 		guest.ExitCleanup(false)

pkg/hostman/guestman/pod.go

@@ -864,7 +864,7 @@ func (s *sPodGuestInstance) getOtherPods() []*sPodGuestInstance {
 	return otherPods
 }
 
-func (s *sPodGuestInstance) getCgroupParent() string {
+func PodCgroupParent() string {
 	return "/cloudpods"
 }
 
@@ -1004,7 +1004,7 @@ func (s *sPodGuestInstance) _startPod(ctx context.Context, userCred mcclient.Tok
 		},
 		Annotations: nil,
 		Linux: &runtimeapi.LinuxPodSandboxConfig{
-			CgroupParent: s.getCgroupParent(),
+			CgroupParent: PodCgroupParent(),
 			SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
 				NamespaceOptions:   s.namespacesForPod(podInput),
 				SelinuxOptions:     nil,
@@ -1696,7 +1696,7 @@ func (s *sPodGuestInstance) getContainerMounts(ctrId string, input *hostapi.Cont
 }
 
 func (s *sPodGuestInstance) getCGUtil() pod.CgroupUtil {
-	return pod.NewPodCgroupV1Util(s.getCgroupParent())
+	return pod.NewPodCgroupV1Util(PodCgroupParent())
 }
 
 func (s *sPodGuestInstance) setContainerCgroupDevicesAllow(ctrId string, allowStrs []string) error {
@@ -2091,6 +2091,24 @@ func (s *sPodGuestInstance) findHostCpuPath(ctrId string, cpuIndex int) (int, er
 	return s.getHostCPUMap().Get(ctrId, cpuIndex)
 }
 
+func (s *sPodGuestInstance) simulateContainerSystemCpuSetScalingCurFreq(ctrId string, scalingCurFreq int64) error {
+	cpuDir := s.getContainerSystemCpusDir(ctrId)
+	cpuCnt := s.GetDesc().Cpu
+	for i := 0; i < int(cpuCnt); i++ {
+		cpufreqPolicyCurFreqFile := path.Join(cpuDir, "cpufreq", fmt.Sprintf("policy%d", i), "scaling_cur_freq")
+		cpufreqPolicySetSpeedFile := path.Join(cpuDir, "cpufreq", fmt.Sprintf("policy%d", i), "scaling_setspeed")
+		scalingCurFreqStr := fmt.Sprintf("%d\n", scalingCurFreq)
+		if err := fileutils2.FilePutContents(cpufreqPolicyCurFreqFile, scalingCurFreqStr, false); err != nil {
+			return errors.Wrapf(err, "failed write %s", cpufreqPolicyCurFreqFile)
+		}
+		if err := fileutils2.FilePutContents(cpufreqPolicySetSpeedFile, scalingCurFreqStr, false); err != nil {
+			return errors.Wrapf(err, "failed write %s", cpufreqPolicySetSpeedFile)
+		}
+	}
+
+	return nil
+}
+
 func (s *sPodGuestInstance) simulateContainerSystemCpu(ctx context.Context, ctrId string) ([]*runtimeapi.Mount, error) {
 	cpuDir := s.getContainerSystemCpusDir(ctrId)
 	cpuCnt := s.GetDesc().Cpu

pkg/hostman/guestman/pod_helper.go

@@ -17,7 +17,10 @@ package guestman
 import (
 	"context"
 	"fmt"
+	"path"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/shirou/gopsutil/v3/disk"
 
@@ -30,6 +33,8 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/container/volume_mount"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/cgrouputils"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/pod/image"
 	"yunion.io/x/onecloud/pkg/util/pod/nerdctl"
 )
@@ -244,3 +249,99 @@ func GetPodStatusByContainerStatus(status string, cStatus string, isPrimary bool
 	}
 	return status
 }
+
+type SCpuFreqRealTimeSimulateManager struct {
+	lastTimeCpuUsage map[string]int64
+	lastTime         *int64
+
+	interval int
+	stop     chan struct{}
+
+	cpufreqMax int64
+	cpufreqMin int64
+}
+
+func newCpuFreqRealTimeSimulateManager(intervalSecond int, cpufreqMax, cpufreqMin int64) *SCpuFreqRealTimeSimulateManager {
+	return &SCpuFreqRealTimeSimulateManager{
+		interval:   intervalSecond,
+		stop:       make(chan struct{}),
+		cpufreqMax: cpufreqMax,
+		cpufreqMin: cpufreqMin,
+	}
+}
+
+func (m *SCpuFreqRealTimeSimulateManager) StartSetCpuFreqSimulate() {
+	log.Infof("StartSetCpuFreqSimulate")
+	ticker := time.NewTicker(time.Duration(m.interval) * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			m.startSetCpuFreqSimulate()
+		case <-m.stop:
+			return
+		}
+	}
+}
+
+func (m *SCpuFreqRealTimeSimulateManager) Stop() {
+	close(m.stop)
+}
+
+func (m *SCpuFreqRealTimeSimulateManager) startSetCpuFreqSimulate() {
+	newCpuUsage := map[string]int64{}
+	startTime := time.Now().UnixNano()
+	guestManager.Servers.Range(func(k, v interface{}) bool {
+		pod, ok := v.(*sPodGuestInstance)
+		if !ok {
+			log.Errorf("is not pod instance")
+			return false
+		}
+		cgroupRoot := path.Join(cgrouputils.GetSubModulePath("cpuacct"), PodCgroupParent())
+		criIds := pod.GetPodContainerCriIds()
+		for i := range criIds {
+			ctr, err := pod.GetContainerByCRIId(criIds[i])
+			if err != nil {
+				log.Errorf("failed get %s ctrid by criId %s", pod.GetName(), criIds[i])
+				continue
+			}
+			cpuDir := pod.getContainerSystemCpusDir(ctr.Id)
+			if !fileutils2.Exists(cpuDir) {
+				log.Errorf("%s %s has no cpuDir", pod.GetName(), criIds[i])
+				continue
+			}
+
+			usagePath := path.Join(cgroupRoot, criIds[i], "cpuacct.usage")
+			criCpuUsageStr, err := fileutils2.FileGetContents(usagePath)
+			if err != nil {
+				continue
+			}
+			criCpuUsageStr = strings.TrimSpace(criCpuUsageStr)
+			criCpuUsage, err := strconv.ParseInt(criCpuUsageStr, 10, 0)
+			if err != nil {
+				log.Errorf("failed parse %s %s: %s", usagePath, criCpuUsageStr, err)
+				continue
+			}
+
+			newCpuUsage[criIds[i]] = criCpuUsage
+			if m.lastTime != nil {
+				if lastTimeUsage, ok := m.lastTimeCpuUsage[criIds[i]]; ok {
+					timeDiff := float64(startTime - *m.lastTime)
+					usageDiff := float64(criCpuUsage - lastTimeUsage)
+					cpuUsagePercent := (usageDiff / timeDiff) * 100
+					freqRange := float64(m.cpufreqMax - m.cpufreqMin)
+					estimatedFreq := m.cpufreqMin + int64(freqRange*cpuUsagePercent/100)
+
+					err := pod.simulateContainerSystemCpuSetScalingCurFreq(ctr.Id, estimatedFreq)
+					if err != nil {
+						log.Errorf("failed set %s(%s) simulate cpufreq: %s", pod.GetId(), criIds[i], err)
+					}
+				}
+			}
+		}
+		return true
+	})
+	m.lastTime = &startTime
+	m.lastTimeCpuUsage = newCpuUsage
+}

pkg/hostman/hostmetrics/container_nvidia_gpu_metrics.go

@@ -147,7 +147,7 @@ func parseNvidiaGpuProcessMetrics(gpuMetricsStr string) []NvidiaGpuProcessMetric
 func (s *SGuestMonitorCollector) collectGpuPodsProcesses() map[string]map[string]struct{} {
 	podProcIds := map[string]map[string]struct{}{}
 	guestmanager := guestman.GetGuestManager()
-	cgroupRoot := path.Join(cgrouputils.GetSubModulePath("cpuset"), "cloudpods")
+	cgroupRoot := path.Join(cgrouputils.GetSubModulePath("cpuset"), guestman.PodCgroupParent())
 	guestmanager.Servers.Range(func(k, v interface{}) bool {
 		pod, ok := v.(guestman.PodInstance)
 		if !ok {

pkg/hostman/options/options.go

@@ -240,6 +240,8 @@ type SHostOptions struct {
 	ContainerDeviceConfigFile                string `help:"container device configuration file path"`
 	LxcfsPath                                string `help:"lxcfs directory path" default:"/var/lib/lxcfs"`
 	ContainerSystemCpufreqSimulateConfigFile string `help:"container system cpu simulate config file path" default:"/etc/yunion/container_cpufreq_simulate.conf"`
+	EnableRealtimeCpufreqSimulate            bool   `help:"realtime cpufreq simulate" default:"true"`
+	RealtimeCpufreqSimulateInterval          int    `help:"realtime cpufreq simulate interval(second)" default:"2"`
 
 	EnableCudaMPS        bool   `help:"enable cuda mps" default:"false"`
 	CudaMPSPipeDirectory string `help:"cuda mps pipe dir" default:"/tmp/nvidia-mps/pipe"`