opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(host): restart pod and containers when crashed (#21348)

Browse Source
This commit is contained in:
Zexi Li
2024-10-06 11:09:23 +08:00
committed by GitHub
parent 5a5631a9a5
commit cc94930e8a
5 changed files with 94 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/host/container.go
+6
View File
@@ -125,3 +125,9 @@ type ContainerCommitInput struct {
Repository string `json:"repository"`
Auth *apis.ContainerPullImageAuthConfig `json:"auth"`
}
type ContainerStopInput struct {
Timeout int64 `json:"timeout"`
ShmSizeMB int `json:"shm_size_mb"`
ContainerName string `json:"container_name"`
}
pkg/hostman/guestman/pod.go
+24 -16
View File
@@ -116,7 +116,7 @@ type PodInstance interface {
DeleteContainer(ctx context.Context, cred mcclient.TokenCredential, id string) (jsonutils.JSONObject, error)
SyncStatus(reason string)
SyncContainerStatus(ctx context.Context, cred mcclient.TokenCredential, ctrId string) (jsonutils.JSONObject, error)
StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error)
StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerStopInput) (jsonutils.JSONObject, error)
PullImage(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerPullImageInput) (jsonutils.JSONObject, error)
SaveVolumeMountToImage(ctx context.Context, userCred mcclient.TokenCredential, input *hostapi.ContainerSaveVolumeMountToImageInput, ctrId string) (jsonutils.JSONObject, error)
ExecContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *computeapi.ContainerExecInput) (*url.URL, error)
@@ -707,6 +707,13 @@ func (s *sPodGuestInstance) StartLocalPod(ctx context.Context, userCred mcclient
return nil
}
func (s *sPodGuestInstance) ShouldRestartPodOnCrash() bool {
if len(s.GetContainers()) <= 1 {
return true
}
return false
}
func (s *sPodGuestInstance) startPod(ctx context.Context, userCred mcclient.TokenCredential) (*computeapi.PodStartResponse, error) {
retries := 3
sec := 5 * time.Second
@@ -753,8 +760,10 @@ func (s *sPodGuestInstance) _startPod(ctx context.Context, userCred mcclient.Tok
LogDirectory: s.getPodLogDir(),
DnsConfig: nil,
PortMappings: nil,
Labels: nil,
Annotations: nil,
Labels: map[string]string{
runtime.PodUIDLabel: s.GetId(),
},
Annotations: nil,
Linux: &runtimeapi.LinuxPodSandboxConfig{
CgroupParent: s.getCgroupParent(),
SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
@@ -1091,7 +1100,7 @@ func (s *sPodGuestInstance) doContainerStartPostLifecycle(ctx context.Context, c
return nil
}
func (s *sPodGuestInstance) StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
func (s *sPodGuestInstance) StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerStopInput) (jsonutils.JSONObject, error) {
criId, err := s.getContainerCRIId(ctrId)
if err != nil {
if errors.Cause(err) == errors.ErrNotFound {
@@ -1104,19 +1113,18 @@ func (s *sPodGuestInstance) StopContainer(ctx context.Context, userCred mcclient
s.expectedStatus.SetContainerStatus(criId, ctrId, computeapi.CONTAINER_STATUS_EXITED)
if body.Contains("timeout") {
timeout, _ = body.Int("timeout")
if input.Timeout != 0 {
timeout = input.Timeout
}
if body.Contains("shm_size_mb") {
shmSizeMB, _ := body.Int("shm_size_mb")
if shmSizeMB > 64 {
name, err := body.GetString("container_name")
if err != nil {
return nil, errors.Wrapf(err, "not found name from body: %s", body)
}
if err := s.unmountDevShm(name); err != nil {
return nil, errors.Wrapf(err, "unmount shm %s", name)
}
shmSizeMB := input.ShmSizeMB
if shmSizeMB > 64 {
name := input.ContainerName
if name == "" {
return nil, errors.Wrapf(errors.ErrNotFound, "not found container_name from input: %s", jsonutils.Marshal(input))
}
if err := s.unmountDevShm(name); err != nil {
return nil, errors.Wrapf(err, "unmount shm %s", name)
}
}
if err := s.getCRI().StopContainer(ctx, criId, timeout); err != nil {
pkg/hostman/guestman/pod_helper.go
+51
View File
@@ -15,6 +15,8 @@
package guestman
import (
"context"
"fmt"
"strings"
"github.com/shirou/gopsutil/v3/disk"
@@ -26,6 +28,7 @@ import (
hostapi "yunion.io/x/onecloud/pkg/apis/host"
"yunion.io/x/onecloud/pkg/hostman/container/volume_mount"
"yunion.io/x/onecloud/pkg/hostman/options"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/pod/image"
"yunion.io/x/onecloud/pkg/util/pod/nerdctl"
)
@@ -148,3 +151,51 @@ func (s *sPodGuestInstance) getVolumeMountUsage(drv volume_mount.IUsageVolumeMou
drv.InjectUsageTags(usage, vol)
return usage, nil
}
func (s *sPodGuestInstance) RestartLocalPodAndContainers(ctx context.Context, cred mcclient.TokenCredential) {
s.manager.GuestStartWorker.Run(newLocalPodRestartTask(ctx, cred, s), nil, nil)
}
type localPodRestartTask struct {
ctx context.Context
userCred mcclient.TokenCredential
pod *sPodGuestInstance
}
func newLocalPodRestartTask(ctx context.Context, userCred mcclient.TokenCredential, pod *sPodGuestInstance) *localPodRestartTask {
return &localPodRestartTask{
ctx: ctx,
userCred: userCred,
pod: pod,
}
}
func (t *localPodRestartTask) Run() {
log.Infof("restart pod and containers locally (%s/%s)", t.pod.Id, t.pod.GetName())
for _, ctr := range t.pod.GetContainers() {
log.Infof("stop container locally (%s/%s/%s/%s)", t.pod.Id, t.pod.GetName(), ctr.Id, ctr.Name)
if _, err := t.pod.StopContainer(t.ctx, t.userCred, ctr.Id, &hostapi.ContainerStopInput{
Timeout: 0,
ShmSizeMB: ctr.Spec.ShmSizeMB,
ContainerName: ctr.Name,
}); err != nil {
log.Errorf("stop container %s error: %v", ctr.Name, err)
}
}
if _, err := t.pod.startPod(t.ctx, t.userCred); err != nil {
log.Errorf("start pod(%s/%s) err: %s", t.pod.GetId(), t.pod.GetName(), err.Error())
return
}
for _, ctr := range t.pod.GetContainers() {
log.Infof("start container locally (%s/%s/%s/%s)", t.pod.Id, t.pod.GetName(), ctr.Id, ctr.Name)
if _, err := t.pod.StartLocalContainer(t.ctx, t.userCred, ctr.Id); err != nil {
log.Errorf("start container %s err: %s", ctr.Id, err.Error())
}
}
t.pod.SyncStatus("sync status after pod and containers restart locally")
}
func (t *localPodRestartTask) Dump() string {
return fmt.Sprintf("pod restart task %s/%s", t.pod.GetId(), t.pod.GetName())
}
pkg/hostman/guestman/pod_sync_loop.go
+8 -4
View File
@@ -88,11 +88,15 @@ func (m *SGuestManager) startContainer(obj *sPodGuestInstance, ctr *hostapi.Cont
reason := fmt.Sprintf("start died container %s when exit code is %d", ctr.Id, cs.ExitCode)
ctx := context.Background()
userCred := hostutils.GetComputeSession(ctx).GetToken()
_, err := obj.StartLocalContainer(ctx, userCred, ctr.Id)
if err != nil {
return errors.Wrap(err, reason)
if obj.ShouldRestartPodOnCrash() {
obj.RestartLocalPodAndContainers(ctx, userCred)
} else {
log.Infof("%s: start local container (%s/%s) success", reason, obj.GetId(), ctr.Name)
_, err := obj.StartLocalContainer(ctx, userCred, ctr.Id)
if err != nil {
return errors.Wrap(err, reason)
} else {
log.Infof("%s: start local container (%s/%s) success", reason, obj.GetId(), ctr.Name)
}
}
return nil
}
pkg/hostman/guestman/podhandlers/podhandlers.go
+5 -1
View File
@@ -178,7 +178,11 @@ func startContainer(ctx context.Context, userCred mcclient.TokenCredential, pod
}
func stopContainer(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
return pod.StopContainer(ctx, userCred, ctrId, body)
input := new(hostapi.ContainerStopInput)
if err := body.Unmarshal(input); err != nil {
return nil, errors.Wrapf(err, "unmarshal to ContainerStopInput: %s", body.String())
}
return pod.StopContainer(ctx, userCred, ctrId, input)
}
func deleteContainer(ctx context.Context, userCred mcclient.TokenCredential, pod guestman.PodInstance, containerId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {