diff --git a/pkg/hostman/guestman/pod_sync_loop.go b/pkg/hostman/guestman/pod_sync_loop.go index 7d7c9d86d8..61095d1231 100644 --- a/pkg/hostman/guestman/pod_sync_loop.go +++ b/pkg/hostman/guestman/pod_sync_loop.go @@ -130,6 +130,10 @@ func (m *SGuestManager) startContainer(obj *sPodGuestInstance, ctr *hostapi.Cont return nil } +func (m *SGuestManager) GetPleg() pleg.PodLifecycleEventGenerator { + return m.pleg +} + func (m *SGuestManager) syncContainerLoop(plegCh chan *pleg.PodLifecycleEvent) { log.Infof("start sync container loop") for { @@ -181,7 +185,7 @@ func (m *SGuestManager) syncContainerLoopIteration(plegCh chan *pleg.PodLifecycl if ctrObj != nil { ccStatus, _, _ = podMan.GetContainerStatus(ctx, ctrObj.Id) } - if !isInternalStopped || ccStatus == computeapi.CONTAINER_STATUS_EXITED { + if !isInternalStopped && ccStatus == computeapi.CONTAINER_STATUS_EXITED { podStatus, err := m.podCache.Get(e.Id) if err != nil { log.Errorf("get pod %s status error: %v", e.Id, err) diff --git a/pkg/util/losetup/ioctl/remove.go b/pkg/util/losetup/ioctl/remove.go index 0fec7caebc..8db282e778 100644 --- a/pkg/util/losetup/ioctl/remove.go +++ b/pkg/util/losetup/ioctl/remove.go @@ -39,8 +39,40 @@ const ( LOOP_CTL_PATH = "/dev/loop-control" LOOP_CTL_REMOVE = C.LOOP_CTL_REMOVE + // LOOP_CTL_GET_FREE = C.LOOP_CTL_GET_FREE + LOOP_CTL_ADD = C.LOOP_CTL_ADD ) +func AddDevice(devNumber int) (string, error) { + fd, err := os.OpenFile(LOOP_CTL_PATH, os.O_RDWR, 0644) + if err != nil { + return "", errors.Wrapf(err, "Open %s", LOOP_CTL_PATH) + } + defer fd.Close() + + retNu, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd.Fd(), LOOP_CTL_ADD, uintptr(devNumber)) + if err != nil && !strings.Contains(err.Error(), "errno 0") { + return "", errors.Wrapf(err, "IOCTL ADD DEVICE") + } + return fmt.Sprintf("/dev/loop%d", retNu), nil +} + +/*func FindFreeDevice() (string, error) { + fd, err := os.OpenFile(LOOP_CTL_PATH, os.O_RDWR, 0644) + if err != nil { + return "", errors.Wrapf(err, "Open %s", LOOP_CTL_PATH) + } + defer fd.Close() + + var devNr int + retNu, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd.Fd(), LOOP_CTL_GET_FREE, uintptr(devNr)) + if err != nil { + return "", errors.Wrapf(err, "IOCTL GET FREE 2") + } + log.Infof("using ioctl get free loop %v", retNu) + return fmt.Sprintf("/dev/loop%d", retNu), nil +}*/ + func RemoveDevice(devNumber int) error { fd, err := os.OpenFile(LOOP_CTL_PATH, os.O_RDWR, 0644) if err != nil { diff --git a/pkg/util/losetup/losetup.go b/pkg/util/losetup/losetup.go index 0a27198743..ec197dde7b 100644 --- a/pkg/util/losetup/losetup.go +++ b/pkg/util/losetup/losetup.go @@ -128,7 +128,7 @@ func listDevicesOldVersion() (*Devices, error) { return strings.TrimSuffix(cmd.Output(), "\n"), nil }*/ -func AttachDevice(filePath string, partScan bool) (*Device, error) { +func attachDevice(devPath string, filePath string, partScan bool) (*Device, error) { // See man-page: https://man7.org/linux/man-pages/man8/losetup.8.html // The loop device setup is not an atomic operation when used with // --find, and losetup does not protect this operation by any lock. @@ -149,8 +149,12 @@ func AttachDevice(filePath string, partScan bool) (*Device, error) { if partScan { args = append(args, "-P") } - // args = append(args, []string{"--find", "--nooverlap", filePath}...) - args = append(args, []string{"--find", filePath}...) + if devPath != "" { + args = append(args, []string{devPath, filePath}...) + } else { + // args = append(args, []string{"--find", "--nooverlap", filePath}...) + args = append(args, []string{"--find", filePath}...) + } _, err = NewLosetupCommand().AddArgs(args...).Run() if err != nil { return nil, err @@ -166,6 +170,14 @@ func AttachDevice(filePath string, partScan bool) (*Device, error) { return dev, nil } +func AttachDeviceWithPath(devPath string, filePath string, partScan bool) (*Device, error) { + return attachDevice(devPath, filePath, partScan) +} + +func AttachDevice(filePath string, partScan bool) (*Device, error) { + return attachDevice("", filePath, partScan) +} + // converts a raw key value pair string into a map of key value pairs // example raw string of `foo="0" bar="1" baz="biz"` is returned as: // map[string]string{"foo":"0", "bar":"1", "baz":"biz"} diff --git a/pkg/util/losetup/manager/loop.go b/pkg/util/losetup/manager/loop.go new file mode 100644 index 0000000000..233396b85f --- /dev/null +++ b/pkg/util/losetup/manager/loop.go @@ -0,0 +1,226 @@ +package manager + +import ( + "fmt" + "math/rand" + "strconv" + "strings" + "sync" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/losetup" + "yunion.io/x/onecloud/pkg/util/losetup/ioctl" + "yunion.io/x/onecloud/pkg/util/mountutils" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +type ILoopDevice interface { + // GetDevicePath 返回设备路径 + GetDevicePath() string + // IsUsed 检查设备是否在使用中 + IsUsed() bool + // SetUsed 设置设备使用状态 + SetUsed(used bool) +} + +type ILoopManager interface { + AttachDevice(filePath string, partScan bool) (*losetup.Device, error) + DetachDevice(devPath string) error +} + +// loopDevice Loop设备实现 +type loopDevice struct { + devicePath string + used bool + lock *sync.Mutex +} + +// NewLoopDevice 创建新的Loop设备 +func NewLoopDevice(devicePath string) ILoopDevice { + return &loopDevice{ + devicePath: devicePath, + used: fileutils2.IsBlockDeviceUsed(devicePath), + lock: &sync.Mutex{}, + } +} + +func (d *loopDevice) GetDevicePath() string { + return d.devicePath +} + +func (d *loopDevice) IsUsed() bool { + return d.used +} + +func (d *loopDevice) SetUsed(used bool) { + d.lock.Lock() + defer d.lock.Unlock() + + d.used = used +} + +func (d *loopDevice) Detach() error { + // 获取所有挂载点 + mountPoints, err := d.getMountPoints() + if err != nil { + return errors.Wrap(err, "getMountPoints") + } + + // 卸载所有挂载点 + for _, mountPoint := range mountPoints { + if err := mountutils.Unmount(mountPoint, false); err != nil { + return errors.Wrapf(err, "umount %s", mountPoint) + } + } + + // 断开loop设备 + if err := losetup.DetachDevice(d.GetDevicePath()); err != nil { + return errors.Wrapf(err, "detach loop device %s", d.GetDevicePath()) + } + + return nil +} + +func (d *loopDevice) getMountPoints() ([]string, error) { + cmd := fmt.Sprintf("mount | grep %sp1 | awk '{print $3}'", d.GetDevicePath()) + output, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", cmd).Output() + if err != nil { + return nil, errors.Wrapf(err, "exec cmd %s: %s", cmd, output) + } + return strings.Split(string(output), "\n"), nil +} + +func (m *loopManager) initDevices() error { + // 使用 ls 和 grep 命令列出所有以数字结尾的 /dev/loop* 设备 + cmd := "ls /dev/loop* | grep -E 'loop[0-9]+$'" + output, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", cmd).Output() + if err != nil { + return errors.Wrap(err, "list loop devices") + } + + // 按行分割输出 + devices := strings.Split(string(output), "\n") + for _, dev := range devices { + dev = strings.TrimSpace(dev) + if dev == "" { + continue + } + m.devices[dev] = NewLoopDevice(dev) + } + + return nil +} + +// loopManager Loop设备管理器 +type loopManager struct { + devices map[string]ILoopDevice + actionLock *sync.Mutex + mapLock *sync.Mutex +} + +// NewLoopManager 创建新的Loop管理器 +func newLoopManager() (ILoopManager, error) { + ret := &loopManager{ + devices: make(map[string]ILoopDevice), + actionLock: &sync.Mutex{}, + mapLock: new(sync.Mutex), + } + + if err := ret.initDevices(); err != nil { + return ret, errors.Wrap(err, "initDevices") + } + return ret, nil +} + +func (m *loopManager) findNewDeviceNumber() (int, error) { + // 获取所有已使用的设备号 + usedNumbers := make(map[int]bool) + for name := range m.devices { + // 从设备名称中提取数字,例如从 /dev/loop0 中提取 0 + numStr := strings.TrimPrefix(name, "/dev/loop") + num, err := strconv.Atoi(numStr) + if err != nil { + return -1, errors.Wrapf(err, "parse device number from %s", name) + } + usedNumbers[num] = true + } + + // 从0开始查找第一个未使用的设备号 + for i := 0; i < 200; i++ { + if !usedNumbers[i] { + // 创建新的loop设备 + return i, nil + } + } + return -1, errors.Wrap(errors.ErrNotFound, "No available device found") +} + +func (m *loopManager) AttachDevice(filePath string, partScan bool) (*losetup.Device, error) { + m.actionLock.Lock() + defer m.actionLock.Unlock() + + dev, err := m.acquireDevice() + if err != nil { + return nil, errors.Wrap(err, "AcquireDevice") + } + loDev, err := losetup.AttachDeviceWithPath(dev.GetDevicePath(), filePath, partScan) + if err != nil { + return nil, errors.Wrapf(err, "AttachDeviceWithPath: %s, filePath: %s", dev.GetDevicePath(), filePath) + } + + return loDev, nil +} + +func (m *loopManager) DetachDevice(devPath string) error { + m.actionLock.Lock() + defer m.actionLock.Unlock() + + if err := losetup.DetachDevice(devPath); err != nil { + return errors.Wrapf(err, "DetachDevice: %s", devPath) + } + m.releaseDevice(devPath) + return nil +} + +func (m *loopManager) acquireDevice() (ILoopDevice, error) { + m.mapLock.Lock() + defer m.mapLock.Unlock() + + freeDevices := make([]ILoopDevice, 0) + for _, device := range m.devices { + if !device.IsUsed() { + freeDevices = append(freeDevices, device) + } + } + if len(freeDevices) > 0 { + device := freeDevices[rand.Intn(len(freeDevices))] + device.SetUsed(true) + return device, nil + } + + // create new device + num, err := m.findNewDeviceNumber() + if err != nil { + return nil, errors.Wrap(err, "findNewDeviceNumber") + } + devPath, err := ioctl.AddDevice(num) + if err != nil { + return nil, errors.Wrapf(err, "add device %d", num) + } + device := NewLoopDevice(devPath) + m.devices[devPath] = device + device.SetUsed(true) + + return device, nil +} + +func (m *loopManager) releaseDevice(devPath string) { + m.mapLock.Lock() + defer m.mapLock.Unlock() + + if dev, ok := m.devices[devPath]; ok { + dev.SetUsed(false) + } +} diff --git a/pkg/util/losetup/manager/loopdev.go b/pkg/util/losetup/manager/manager.go similarity index 91% rename from pkg/util/losetup/manager/loopdev.go rename to pkg/util/losetup/manager/manager.go index c9999462b9..536cbd8f4c 100644 --- a/pkg/util/losetup/manager/loopdev.go +++ b/pkg/util/losetup/manager/manager.go @@ -97,11 +97,17 @@ func newDetachOption(devPath string) *execOption { type manager struct { execOptCh chan *execOption + loopMan ILoopManager } func newManager() iManager { + loopMan, err := newLoopManager() + if err != nil { + log.Fatalf("new loop manager error: %v", err) + } m := &manager{ execOptCh: make(chan *execOption), + loopMan: loopMan, } go func() { m.startExec() @@ -132,14 +138,14 @@ func (m *manager) startExec() { switch execOpt.Type { case EXEC_ATTACH: log.Infof("attach %s", jsonutils.Marshal(execOpt.Attach)) - dev, err := losetup.AttachDevice(execOpt.Attach.FilePath, execOpt.Attach.PartScan) + dev, err := m.loopMan.AttachDevice(execOpt.Attach.FilePath, execOpt.Attach.PartScan) execOpt.result <- execResult{ attachedDevice: dev, error: err, } case EXEC_DETACH: log.Infof("detach %s", jsonutils.Marshal(execOpt.Detach)) - err := losetup.DetachDevice(execOpt.Detach.DevicePath) + err := m.loopMan.DetachDevice(execOpt.Detach.DevicePath) execOpt.result <- execResult{ error: err, }