opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-19 10:46:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(host): add lock when attach loop device (#21453)

Browse Source 
* fix(host): add lock when attach loop device

* fix(host): only using reconcileContainerLoop to manage containers
This commit is contained in:
Zexi Li
2024-10-24 16:09:20 +08:00
committed by GitHub
parent 9cdc4fd6bb
commit 0b8ae5edb3
5 changed files with 52 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/hostman/guestman/guestman.go
+12 -6
View File
@@ -152,16 +152,22 @@ func NewGuestManager(host hostutils.IHost, serversPath string, workerCnt int) (*
manager.containerRuntimeManager = runtimeMan
manager.pleg = pleg.NewGenericPLEG(runtimeMan, pleg.ChannelCapacity, pleg.RelistPeriod, manager.podCache, clock.RealClock{})
manager.pleg.Start()
go func() {
manager.syncContainerLoop(manager.pleg.Watch())
}()
go func() {
manager.reconcileContainerLoop(manager.podCache)
}()
manager.startContainerSyncLoop()
}
return manager, nil
}
func (m *SGuestManager) startContainerSyncLoop() {
if m.host.IsContainerHost() {
go func() {
m.syncContainerLoop(m.pleg.Watch())
}()
go func() {
m.reconcileContainerLoop(m.podCache)
}()
}
}
func (m *SGuestManager) InitQemuMaxCpus(machineCaps []monitor.MachineInfo, kvmMaxCpus uint) {
m.qemuMachineCpuMax[compute.VM_MACHINE_TYPE_PC] = arch.X86_MAX_CPUS
m.qemuMachineCpuMax[compute.VM_MACHINE_TYPE_Q35] = arch.X86_MAX_CPUS
pkg/hostman/guestman/pod.go
+7 -11
View File
@@ -273,11 +273,12 @@ func (s *sPodGuestInstance) ImportServer(pendingDelete bool) {
s.manager.SaveServer(s.Id, s)
s.manager.RemoveCandidateServer(s)
if s.IsDaemon() || s.IsDirtyShutdown() {
ctx := context.Background()
/*ctx := context.Background()
cred := hostutils.GetComputeSession(ctx).GetToken()
if err := s.StartLocalPod(ctx, cred); err != nil {
log.Errorf("start local pod err %s", err.Error())
}
}*/
log.Warningf("pod %s need started, waiting sync loop to manage it", s.GetName())
} else {
s.SyncStatus("sync status after host started")
s.getProbeManager().AddPod(s.Desc)
@@ -1156,22 +1157,17 @@ func (s *sPodGuestInstance) StopContainer(ctx context.Context, userCred mcclient
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := s.getCRI().StopContainer(ctx, criId, timeout); err != nil {
if err := s.getCRI().StopContainer(ctx, criId, timeout, true); err != nil {
if !IsContainerNotFoundError(err) {
return nil, errors.Wrap(err, "CRI.StopContainer")
} else {
log.Warningf("CRI.StopContainer %s not found", criId)
}
}
select {
case <-ctx.Done():
return nil, errors.Wrap(ctx.Err(), "stop container")
default:
if err := s.startStat.RemoveContainerFile(ctrId); err != nil {
return nil, errors.Wrap(err, "startStat.RemoveContainerFile")
}
return nil, nil
if err := s.startStat.RemoveContainerFile(ctrId); err != nil {
return nil, errors.Wrap(err, "startStat.RemoveContainerFile")
}
return nil, nil
}
func (s *sPodGuestInstance) GetCRIId() string {
pkg/util/losetup/device.go
+6 -4
View File
@@ -77,18 +77,20 @@ func parseDevice(line string) (Device, error) {
}
func (devs Devices) GetDeviceByName(name string) *Device {
for _, dev := range devs.LoopDevs {
for i := range devs.LoopDevs {
dev := &devs.LoopDevs[i]
if dev.Name == name {
return &dev
return dev
}
}
return nil
}
func (devs Devices) GetDeviceByFile(filePath string) *Device {
for _, dev := range devs.LoopDevs {
for i := range devs.LoopDevs {
dev := &devs.LoopDevs[i]
if dev.BackFile == filePath {
return &dev
return dev
}
}
return nil
pkg/util/losetup/losetup.go
+14 -3
View File
@@ -19,6 +19,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
@@ -32,6 +33,10 @@ const (
LOSETUP_COMMAND = "losetup"
)
var (
attachDeviceLock = sync.Mutex{}
)
type Command struct {
Path string
Args []string
@@ -114,16 +119,22 @@ func listDevicesOldVersion() (*Devices, error) {
return devs, err
}
func GetUnusedDevice() (string, error) {
/*func GetUnusedDevice() (string, error) {
// find first unused device
cmd, err := NewLosetupCommand().AddArgs("-f").Run()
if err != nil {
return "", err
}
return strings.TrimSuffix(cmd.Output(), "\n"), nil
}
}*/
func AttachDevice(filePath string, partScan bool) (*Device, error) {
// See man-page: https://man7.org/linux/man-pages/man8/losetup.8.html
// The loop device setup is not an atomic operation when used with
// --find, and losetup does not protect this operation by any lock.
attachDeviceLock.Lock()
defer attachDeviceLock.Unlock()
oldDevs, err := ListDevices()
if err != nil {
return nil, err
@@ -138,7 +149,7 @@ func AttachDevice(filePath string, partScan bool) (*Device, error) {
if partScan {
args = append(args, "-P")
}
args = append(args, []string{"-f", filePath}...)
args = append(args, []string{"--find", "--nooverlap", filePath}...)
_, err = NewLosetupCommand().AddArgs(args...).Run()
if err != nil {
return nil, err
pkg/util/pod/pod.go
+13 -3
View File
@@ -36,7 +36,7 @@ type CRI interface {
RemovePod(ctx context.Context, podId string) error
CreateContainer(ctx context.Context, podId string, podConfig *runtimeapi.PodSandboxConfig, ctrConfig *runtimeapi.ContainerConfig, withPull bool) (string, error)
StartContainer(ctx context.Context, id string) error
StopContainer(ctx context.Context, ctrId string, timeout int64) error
StopContainer(ctx context.Context, ctrId string, timeout int64, tryRemove bool) error
RemoveContainer(ctx context.Context, ctrId string) error
RunContainers(ctx context.Context, podConfig *runtimeapi.PodSandboxConfig, containerConfigs []*runtimeapi.ContainerConfig, runtimeHandler string) (*RunContainersResponse, error)
ListContainers(ctx context.Context, opts ListContainerOptions) ([]*runtimeapi.Container, error)
@@ -329,7 +329,7 @@ func (c crictl) RemovePod(ctx context.Context, podId string) error {
return nil
}
func (c crictl) StopContainer(ctx context.Context, ctrId string, timeout int64) error {
func (c crictl) StopContainer(ctx context.Context, ctrId string, timeout int64, tryRemove bool) error {
maxTries := 5
interval := 3 * time.Second
errs := []error{}
@@ -339,10 +339,20 @@ func (c crictl) StopContainer(ctx context.Context, ctrId string, timeout int64)
return nil
}
dur := interval * time.Duration(tries+1)
log.Warningf("try to restop container %s after %s: %v", ctrId, dur, err)
log.Warningf("try to restop container %s after %s, timeout: %d: %v", ctrId, dur, timeout, err)
// set timeout to 0 to stop forcely
timeout = 0
errs = append(errs, errors.Wrapf(err, "try %d", tries))
time.Sleep(dur)
}
if tryRemove {
// try force remove container
if err := c.RemoveContainer(ctx, ctrId); err != nil {
errs = append(errs, errors.Wrapf(err, "try remove container %s", ctrId))
} else {
return nil
}
}
return errors.NewAggregate(errs)
}