cloudpods/pkg/util/pod/cpus.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pod

import (
	"fmt"
	"path/filepath"
	"strconv"
	"sync"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"

	hostapi "yunion.io/x/onecloud/pkg/apis/host"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
)

func EnsureDir(dir string) error {
	out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output()
	if err != nil {
		return errors.Wrapf(err, "mkdir %s: %s", dir, out)
	}
	return nil
}

func EnsureFile(filename string, content string, mod string) error {
	cmds := []string{
		fmt.Sprintf("echo '%s' > %s", content, filename),
	}
	if mod != "" {
		cmds = append(cmds, fmt.Sprintf("chmod %s %s", mod, filename))
	}
	for _, cmd := range cmds {
		out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", cmd).Output()
		if err != nil {
			return errors.Wrapf(err, "cmd %s: %s", cmd, out)
		}
	}
	return nil
}

func EnsureContainerSystemCpuDir(cpuDir string, cpuCount int64) error {
	if err := EnsureDir(cpuDir); err != nil {
		return err
	}
	for i := 0; i < int(cpuCount); i++ {
		singleCpuDir := filepath.Join(cpuDir, fmt.Sprintf("cpu%d", i))
		if err := EnsureDir(singleCpuDir); err != nil {
			return errors.Wrapf(err, "create cpu%d", i)
		}
	}
	// create other dirs
	freqDir := filepath.Join(cpuDir, "cpufreq")
	idleDir := filepath.Join(cpuDir, "cpuidle")
	hotplugDir := filepath.Join(cpuDir, "hotplug")
	powerDir := filepath.Join(cpuDir, "power")
	for _, dir := range []string{freqDir, idleDir, hotplugDir, powerDir} {
		if err := EnsureDir(dir); err != nil {
			return err
		}
	}
	// create files
	if err := EnsureFile(filepath.Join(cpuDir, "isolated"), "", "755"); err != nil {
		return err
	}
	if err := EnsureFile(filepath.Join(cpuDir, "kernel_max"), fmt.Sprintf("%d", cpuCount), "644"); err != nil {
		return err
	}
	if err := EnsureFile(filepath.Join(cpuDir, "modalias"), "", "755"); err != nil {
		return err
	}
	if err := EnsureFile(filepath.Join(cpuDir, "offline"), "", "755"); err != nil {
		return err
	}
	ensureCpuRangeFile := func(baseName string, mode string) error {
		if mode == "" {
			mode = "644"
		}
		if cpuCount > 1 {
			if err := EnsureFile(filepath.Join(cpuDir, baseName), fmt.Sprintf("0-%d", cpuCount-1), mode); err != nil {
				return err
			}
			return nil
		}
		if err := EnsureFile(filepath.Join(cpuDir, baseName), "0", mode); err != nil {
			return err
		}
		return nil
	}

	if err := ensureCpuRangeFile("online", "755"); err != nil {
		return err
	}
	if err := ensureCpuRangeFile("possible", ""); err != nil {
		return err
	}
	if err := ensureCpuRangeFile("present", ""); err != nil {
		return err
	}
	if err := EnsureFile(filepath.Join(cpuDir, "uevent"), "", "755"); err != nil {
		return err
	}
	return nil
}

type ContainerCPU struct {
	ContainerId string `json:"container_id"`
	Index       int    `json:"index"`
}

func NewContainerCPU(ctrId string, ctrIdx int) *ContainerCPU {
	return &ContainerCPU{
		ContainerId: ctrId,
		Index:       ctrIdx,
	}
}

type HostContainerCPU struct {
	Index      int                        `json:"index"`
	Containers map[string][]*ContainerCPU `json:"containers"`
}

func NewHostContainerCPU(hostIndex int) *HostContainerCPU {
	return &HostContainerCPU{
		Index:      hostIndex,
		Containers: make(map[string][]*ContainerCPU),
	}
}

func (h *HostContainerCPU) HasContainer(ctrId string) bool {
	_, ok := h.Containers[ctrId]
	return ok
}

func (h *HostContainerCPU) DeleteContainer(ctrId string) {
	delete(h.Containers, ctrId)
}

func (h *HostContainerCPU) InsertContainer(ctrId string, ctrIdx int) {
	if h.Containers == nil {
		h.Containers = make(map[string][]*ContainerCPU)
	}
	if h.HasContainer(ctrId) {
		h.Containers[ctrId] = append(h.Containers[ctrId], NewContainerCPU(ctrId, ctrIdx))
	} else {
		h.Containers[ctrId] = []*ContainerCPU{NewContainerCPU(ctrId, ctrIdx)}
	}
}

var (
	hostContainerCPUMapLock = sync.Mutex{}
)

type HostContainerCPUMap struct {
	Map       map[string]*HostContainerCPU `json:"map"`
	stateFile string
}

func NewHostContainerCPUMap(topo *hostapi.HostTopology, stateFile string) (*HostContainerCPUMap, error) {
	ret := make(map[string]*HostContainerCPU)
	if fileutils2.Exists(stateFile) {
		content, err := fileutils2.FileGetContents(stateFile)
		if err != nil {
			return nil, errors.Wrapf(err, "get file contents: %s", stateFile)
		}
		obj, err := jsonutils.ParseString(content)
		if err != nil {
			return nil, errors.Wrapf(err, "parse to json: %s", content)
		}
		hm := new(HostContainerCPUMap)
		if err := obj.Unmarshal(hm); err != nil {
			return nil, errors.Wrap(err, "unmarshal to HostContainerCPUMap")
		}
		hm.stateFile = stateFile
		return hm, nil
	}
	nodes := topo.Nodes
	for _, node := range nodes {
		for _, core := range node.Cores {
			for _, processor := range core.LogicalProcessors {
				ret[fmt.Sprintf("%d", processor)] = NewHostContainerCPU(processor)
			}
		}
	}
	return &HostContainerCPUMap{Map: ret, stateFile: stateFile}, nil
}

func (hm *HostContainerCPUMap) dumpToFile() error {
	return fileutils2.FilePutContents(hm.stateFile, jsonutils.Marshal(hm).PrettyString(), false)
}

func (hm *HostContainerCPUMap) Delete(ctrId string) error {
	hostContainerCPUMapLock.Lock()
	defer hostContainerCPUMapLock.Unlock()

	for _, cm := range hm.Map {
		if cm.HasContainer(ctrId) {
			cm.DeleteContainer(ctrId)
		}
	}
	return hm.dumpToFile()
}

func (hm *HostContainerCPUMap) Get(ctrId string, ctrCpuIndex int) (int, error) {
	hostContainerCPUMapLock.Lock()
	defer hostContainerCPUMapLock.Unlock()

	hostIndex := hm.findLeastUsedIndex(ctrId, ctrCpuIndex)
	if err := hm.markUsed(hostIndex, ctrId, ctrCpuIndex); err != nil {
		return 0, errors.Wrapf(err, "mark container %s %d used", ctrId, ctrCpuIndex)
	}
	return hostIndex, nil
}

func (hm *HostContainerCPUMap) findLeastUsedIndex(ctrId string, ctrCpuIndex int) int {
	unusedMap := make(map[string]*HostContainerCPU)
	usedMap := make(map[string]*HostContainerCPU)
	for idx, hc := range hm.Map {
		tmpHc := hc
		if tmpHc.HasContainer(ctrId) {
			usedMap[idx] = tmpHc
		} else {
			unusedMap[idx] = tmpHc
		}
	}
	if len(unusedMap) != 0 {
		for idx, _ := range unusedMap {
			idxNum, _ := strconv.Atoi(idx)
			return idxNum
		}
	}
	// find the least used index from usedMap
	isStart := true
	leastIndex := 0
	leastUsed := 0
	for idx, hc := range usedMap {
		cm := hc.Containers[ctrId]
		idxNum, _ := strconv.Atoi(idx)
		if isStart {
			leastIndex = idxNum
			leastUsed = len(cm)
			isStart = false
			continue
		}
		if len(cm) < leastUsed {
			leastUsed = len(cm)
			leastIndex = idxNum
		}
	}
	return leastIndex
}

func (hm *HostContainerCPUMap) markUsed(hostIdx int, ctrId string, ctrIdx int) error {
	hc := hm.Map[fmt.Sprintf("%d", hostIdx)]
	hc.InsertContainer(ctrId, ctrIdx)
	return hm.dumpToFile()
}