opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-08-30 17:13:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(host): support container always_restart (#22403)

Browse Source 
* fix(container): create steam app

* feat(host): support container always_restart
This commit is contained in:
Zexi Li
2025-04-11 17:08:02 +08:00
committed by GitHub
parent feb37fd5b1
commit 195666dfeb
4 changed files with 106 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/container-examples/create_steam.go
+88 -38
View File
@@ -10,6 +10,7 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/util/rand"
api "yunion.io/x/onecloud/pkg/apis"
"yunion.io/x/onecloud/pkg/apis/compute"
@@ -58,6 +59,7 @@ var (
enableLxcfs bool
gpu string
gpuEnvId string
renderNode string
overlay string
alwaysMountDriverVol bool
@@ -66,7 +68,12 @@ var (
devsList []string
wolfAllGpu bool
wolfAllGpu bool
mounts string
mountList []string
appMounts string
appMountList []string
steamNoBigScreen bool
)
func init() {
@@ -74,23 +81,27 @@ func init() {
flag.StringVar(&user, "user", "", "user")
flag.StringVar(&password, "password", "", "password")
flag.StringVar(&region, "region", "", "region")
flag.StringVar(&podNet, "net", "cgame", "pod net")
flag.StringVar(&podIP, "ip", "192.168.6.70", "pod ip")
flag.StringVar(&podNet, "net", "", "pod net")
flag.StringVar(&podIP, "ip", "", "pod ip")
flag.StringVar(&podName, "name", "steam", "pod name")
flag.IntVar(&ncpu, "ncpu", 8, "cpu count")
flag.IntVar(&mem, "mem", 16, "memory in GB")
flag.IntVar(&diskSizeGB, "disk-size", 10, "disk size in GB")
flag.IntVar(&accessPort, "port", 20105, "moonlight access http port")
flag.StringVar(&wolfImage, "wolf-image", "registry.cn-beijing.aliyuncs.com/zexi/wolf:ps5-hook-0401.0", "wolf image")
flag.StringVar(&steamImage, "steam-image", "registry.cn-beijing.aliyuncs.com/zexi/steam:custom.2", "steam image")
flag.StringVar(&wolfImage, "wolf-image", "registry.cn-beijing.aliyuncs.com/zexi/wolf:hook-0408.0", "wolf image")
flag.StringVar(&steamImage, "steam-image", "registry.cn-beijing.aliyuncs.com/zexi/steam:custom.3", "steam image")
flag.StringVar(&externalIP, "eip", "", "external ip")
flag.BoolVar(&enableLxcfs, "lxcfs", false, "enable lxcfs")
flag.BoolVar(&alwaysMountDriverVol, "mount-driver-vol", false, "always mount driver volume")
flag.StringVar(&gpu, "gpu", "", "gpu")
flag.StringVar(&gpuEnvId, "gpu-env-id", "", "gpu env id")
flag.StringVar(&renderNode, "render-node", "/dev/dri/renderD128", "render node")
flag.StringVar(&overlay, "overlay", "", "overlay")
flag.StringVar(&devs, "devs", "", "devs")
flag.StringVar(&mounts, "mounts", "", "mounts")
flag.StringVar(&appMounts, "app-mounts", "", "app mounts")
flag.BoolVar(&wolfAllGpu, "wolf-all-gpu", false, "wolf all gpu")
flag.BoolVar(&steamNoBigScreen, "steam-no-big-screen", false, "steam no big screen")
flag.Parse()
wolfBasePort = accessPort - 5
@@ -98,6 +109,8 @@ func init() {
log.Infof("Connecting to %s as %s", authUrl, user)
devsList = strings.Split(devs, ",")
mountList = strings.Split(mounts, ",")
appMountList = strings.Split(appMounts, ",")
}
func initAuthInfo() {
@@ -133,6 +146,27 @@ func NewEnv(key, val string) *api.ContainerKeyValue {
}
}
func getMounts(mountList []string) []*api.ContainerVolumeMount {
ret := make([]*api.ContainerVolumeMount, len(mountList))
for i, m := range mountList {
parts := strings.Split(m, ":")
if len(parts) != 2 {
log.Fatalf("Invalid mount spec: %s", m)
}
uniqName := fmt.Sprintf("%s_%s", m, rand.String(2))
ret[i] = &api.ContainerVolumeMount{
UniqueName: uniqName,
Type: api.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH,
MountPath: parts[1],
HostPath: &api.ContainerVolumeMountHostPath{
Type: api.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_FILE,
Path: parts[0],
},
}
}
return ret
}
func getTmpSocketsHostPath(name string) string {
return fmt.Sprintf("/tmp/%s/sockets", name)
}
@@ -203,7 +237,7 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput {
eip = externalIP
}
envs := []*api.ContainerKeyValue{
NewEnv("WOLF_LOG_LEVEL", "DEBUG"),
// NewEnv("WOLF_LOG_LEVEL", "DEBUG"),
NewEnv("WOLF_BASE_PORT", fmt.Sprintf("%d", wolfBasePort)),
NewEnv("WOLF_EXTERNAL_IP", eip),
NewEnv("HOST_APPS_STATE_FOLDER", "/etc/wolf"),
@@ -223,17 +257,21 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput {
NewHostDev(DEV_UINPUT),
NewHostDev(DEV_UHID),
}
if gpu == "" || wolfAllGpu {
devs = append(devs, NewHostDev(DEV_DRI))
devs = append(devs, getNvidiaNvDevs(0)...)
if gpu == "" {
if !wolfAllGpu {
devs = append(devs, NewHostDev(DEV_DRI))
devs = append(devs, getNvidiaNvDevs(0)...)
}
} else {
id0 := 0
devs = append(devs, &compute.ContainerDevice{
Type: api.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,
IsolatedDevice: &compute.ContainerIsolatedDevice{
Index: &id0,
},
})
if !wolfAllGpu {
id0 := 0
devs = append(devs, &compute.ContainerDevice{
Type: api.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,
IsolatedDevice: &compute.ContainerIsolatedDevice{
Index: &id0,
},
})
}
}
vms := []*api.ContainerVolumeMount{
/*{
@@ -263,7 +301,7 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput {
Type: api.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_DIRECTORY,
Path: VOL_RUN_UDEV,
},
Propagation: api.MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL,
//Propagation: api.MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL,
},
{
UniqueName: "dev",
@@ -273,7 +311,8 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput {
Type: api.CONTAINER_VOLUME_MOUNT_HOST_PATH_TYPE_DIRECTORY,
Path: VOL_DEV,
},
Propagation: api.MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL,
// WARN: 这里不能用 bidirectional mount
// Propagation: api.MOUNTPROPAGATION_PROPAGATION_BIDIRECTIONAL,
},
{
UniqueName: "tmp-sockets",
@@ -306,6 +345,7 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput {
},
},
}
vms = append(vms, getMounts(mountList)...)
if gpu == "" {
vms = append(vms,
&api.ContainerVolumeMount{
@@ -336,7 +376,7 @@ func NewWolfContainer(gpu string) *compute.PodContainerCreateInput {
}
}
func NewAppSteamContainer(gpu string) *compute.PodContainerCreateInput {
func NewAppSteamContainer(gpu, gpuEnvId string) *compute.PodContainerCreateInput {
// TODO: 设置 ulimit 和 ipc host
// --ipc host --ulimit nofile=10240:10240
zero := 0
@@ -345,8 +385,30 @@ func NewAppSteamContainer(gpu string) *compute.PodContainerCreateInput {
NewHostDev(DEV_UINPUT),
NewHostDev(DEV_UHID),
}
if gpu == "" {
envs := []*api.ContainerKeyValue{
NewEnv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"),
NewEnv("UNAME", "retro"),
NewEnv("UMASK", "000"),
NewEnv("HOME", "/home/retro"),
NewEnv("TZ", "Europe/London"),
NewEnv("NEEDRESTART_SUSPEND", "1"),
NewEnv("GAMESCOPE_VERSION", "3.15.14"),
NewEnv("BUILD_ARCHITECTURE", "amd64"),
NewEnv("DEBIAN_FRONTEND", "noninteractive"),
NewEnv("DEB_BUILD_OPTIONS", "noddeb"),
NewEnv("XDG_RUNTIME_DIR", "/tmp/sockets"),
}
if steamNoBigScreen {
envs = append(envs, NewEnv("STEAM_STARTUP_FLAGS", "-fullscreen"))
}
if gpu == "" && gpuEnvId == "" {
devs = append(devs, getNvidiaAppDevs(0)...)
} else if gpuEnvId != "" {
envs = append(envs,
NewEnv("NVIDIA_VISIBLE_DEVICES", gpuEnvId),
NewEnv("NVIDIA_DRIVER_CAPABILITIES", "all"))
} else {
devs = append(devs, &compute.ContainerDevice{
Type: api.CONTAINER_DEVICE_TYPE_ISOLATED_DEVICE,
@@ -366,10 +428,9 @@ func NewAppSteamContainer(gpu string) *compute.PodContainerCreateInput {
},
}
if overlay != "" {
overlayParts := strings.Split(overlay, ":")
dataVol.Disk.Overlay = &api.ContainerVolumeMountDiskOverlay{
LowerDir: []string{
overlay,
},
LowerDir: overlayParts,
}
}
@@ -423,6 +484,8 @@ func NewAppSteamContainer(gpu string) *compute.PodContainerCreateInput {
},
},
}
vols = append(vols, getMounts(mountList)...)
vols = append(vols, getMounts(appMountList)...)
if gpu == "" || alwaysMountDriverVol {
vols = append(vols,
&api.ContainerVolumeMount{
@@ -452,20 +515,7 @@ func NewAppSteamContainer(gpu string) *compute.PodContainerCreateInput {
CGROUP_RULE_13,
CGROUP_RULE_244,
},
Envs: []*api.ContainerKeyValue{
NewEnv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"),
NewEnv("UNAME", "retro"),
NewEnv("UMASK", "000"),
NewEnv("HOME", "/home/retro"),
NewEnv("TZ", "Europe/London"),
NewEnv("NEEDRESTART_SUSPEND", "1"),
NewEnv("GAMESCOPE_VERSION", "3.15.14"),
NewEnv("BUILD_ARCHITECTURE", "amd64"),
NewEnv("DEBIAN_FRONTEND", "noninteractive"),
NewEnv("DEB_BUILD_OPTIONS", "noddeb"),
NewEnv("XDG_RUNTIME_DIR", "/tmp/sockets"),
// NewEnv("STEAM_STARTUP_FLAGS", "-fullscreen"),
},
Envs: envs,
Capabilities: &api.ContainerCapability{
Add: []string{"SYS_ADMIN", "SYS_NICE", "SYS_PTRACE", "NET_RAW", "MKNOD", "NET_ADMIN"},
},
@@ -553,7 +603,7 @@ func GetCreateParams(name string) *compute.ServerCreateInput {
Containers: []*compute.PodContainerCreateInput{
NewPulseAudioContainer(),
NewWolfContainer(gpu),
NewAppSteamContainer(gpu),
NewAppSteamContainer(gpu, gpuEnvId),
},
}
if gpu != "" {
pkg/apis/container.go
+2 -1
View File
@@ -116,7 +116,8 @@ type ContainerSpec struct {
//LivenessProbe *ContainerProbe `json:"liveness_probe,omitempty"`
// StartupProbe indicates that the Pod has successfully initialized.
// If specified, no other probes are executed until this completes successfully.
StartupProbe *ContainerProbe `json:"startup_probe,omitempty"`
StartupProbe *ContainerProbe `json:"startup_probe,omitempty"`
AlwaysRestart bool `json:"always_restart"`
}
func (c *ContainerSpec) NeedProbe() bool {
pkg/hostman/guestman/pod.go
+6 -1
View File
@@ -220,7 +220,8 @@ type sPodGuestInstance struct {
startStat *startStatHelper
expectedStatus *PodExpectedStatus
startPodLock sync.Mutex
startPodLock sync.Mutex
saveContainerLock sync.Mutex
}
func newPodGuestInstance(id string, man *SGuestManager) PodInstance {
@@ -228,6 +229,7 @@ func newPodGuestInstance(id string, man *SGuestManager) PodInstance {
sBaseGuestInstance: newBaseGuestInstance(id, man, computeapi.HYPERVISOR_POD),
containers: make(map[string]*sContainer),
startPodLock: sync.Mutex{},
saveContainerLock: sync.Mutex{},
}
es, err := NewPodExpectedStatus(p.HomeDir(), computeapi.VM_UNKNOWN)
if err != nil {
@@ -1392,6 +1394,9 @@ func (s *sPodGuestInstance) GetPortMappings() (computeapi.GuestPortMappings, err
}
func (s *sPodGuestInstance) saveContainer(id string, criId string) error {
s.saveContainerLock.Lock()
defer s.saveContainerLock.Unlock()
_, ok := s.containers[id]
if ok {
return errors.Errorf("container %s already exists", criId)
pkg/hostman/guestman/pod_sync_loop.go
+10 -6
View File
@@ -50,7 +50,7 @@ func (m *SGuestManager) reconcileContainerLoop(cache runtime.Cache) {
}
return true
})
time.Sleep(10 * time.Second)
time.Sleep(5 * time.Second)
}
}
@@ -99,18 +99,22 @@ func (m *SGuestManager) startContainer(obj *sPodGuestInstance, ctr *hostapi.Cont
step := 5 * time.Second
internal := time.Duration(int(step) * (attempt * attempt))
curInternal := time.Now().Sub(finishedAt)
if curInternal < internal {
log.Infof("current internal time (%s) < crash_back_off time (%s), skipping restart container(%s/%s)", curInternal, internal, obj.GetId(), ctr.Name)
return nil
if !ctr.Spec.AlwaysRestart {
if curInternal < internal {
log.Infof("current internal time (%s) < crash_back_off time (%s), skipping restart container(%s/%s)", curInternal, internal, obj.GetId(), ctr.Name)
return nil
} else {
log.Infof("current internal time (%s | %s) > crash_back_off time (%s), restart container(%s/%s)", finishedAt, curInternal, internal, obj.GetId(), ctr.Name)
}
} else {
log.Infof("current internal time (%s | %s) > crash_back_off time (%s), restart container(%s/%s)", finishedAt, curInternal, internal, obj.GetId(), ctr.Name)
log.Infof("always restart container(%s/%s) ...", obj.GetId(), ctr.Name)
}
reason := fmt.Sprintf("start died container %s when exit code is %d", ctr.Id, cs.ExitCode)
ctx := context.Background()
userCred := hostutils.GetComputeSession(ctx).GetToken()
if obj.ShouldRestartPodOnCrash() {
// FIXME: 目前不用 workser 来后台异步运行 pod restart task
// FIXME: 目前不用 worker 来后台异步运行 pod restart task
// 这里异步运行会导致容器如果在 10s 没启动完成，又会进行新一轮排队
// 所以改成同步串行执行
//obj.RestartLocalPodAndContainers(ctx, userCred)