feat(host): loop device manager (#22590)

pkg/util/losetup/ioctl/remove.go

@@ -39,8 +39,40 @@ const (
 	LOOP_CTL_PATH = "/dev/loop-control"
 
 	LOOP_CTL_REMOVE = C.LOOP_CTL_REMOVE
+	// LOOP_CTL_GET_FREE = C.LOOP_CTL_GET_FREE
+	LOOP_CTL_ADD = C.LOOP_CTL_ADD
 )
 
+func AddDevice(devNumber int) (string, error) {
+	fd, err := os.OpenFile(LOOP_CTL_PATH, os.O_RDWR, 0644)
+	if err != nil {
+		return "", errors.Wrapf(err, "Open %s", LOOP_CTL_PATH)
+	}
+	defer fd.Close()
+
+	retNu, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd.Fd(), LOOP_CTL_ADD, uintptr(devNumber))
+	if err != nil && !strings.Contains(err.Error(), "errno 0") {
+		return "", errors.Wrapf(err, "IOCTL ADD DEVICE")
+	}
+	return fmt.Sprintf("/dev/loop%d", retNu), nil
+}
+
+/*func FindFreeDevice() (string, error) {
+	fd, err := os.OpenFile(LOOP_CTL_PATH, os.O_RDWR, 0644)
+	if err != nil {
+		return "", errors.Wrapf(err, "Open %s", LOOP_CTL_PATH)
+	}
+	defer fd.Close()
+
+	var devNr int
+	retNu, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd.Fd(), LOOP_CTL_GET_FREE, uintptr(devNr))
+	if err != nil {
+		return "", errors.Wrapf(err, "IOCTL GET FREE 2")
+	}
+	log.Infof("using ioctl get free loop %v", retNu)
+	return fmt.Sprintf("/dev/loop%d", retNu), nil
+}*/
+
 func RemoveDevice(devNumber int) error {
 	fd, err := os.OpenFile(LOOP_CTL_PATH, os.O_RDWR, 0644)
 	if err != nil {

pkg/util/losetup/losetup.go

@@ -128,7 +128,7 @@ func listDevicesOldVersion() (*Devices, error) {
 	return strings.TrimSuffix(cmd.Output(), "\n"), nil
 }*/
 
-func AttachDevice(filePath string, partScan bool) (*Device, error) {
+func attachDevice(devPath string, filePath string, partScan bool) (*Device, error) {
 	// See man-page: https://man7.org/linux/man-pages/man8/losetup.8.html
 	// The loop device setup is not an atomic operation when used with
 	// --find, and losetup does not protect this operation by any lock.
@@ -149,8 +149,12 @@ func AttachDevice(filePath string, partScan bool) (*Device, error) {
 	if partScan {
 		args = append(args, "-P")
 	}
-	// args = append(args, []string{"--find", "--nooverlap", filePath}...)
-	args = append(args, []string{"--find", filePath}...)
+	if devPath != "" {
+		args = append(args, []string{devPath, filePath}...)
+	} else {
+		// args = append(args, []string{"--find", "--nooverlap", filePath}...)
+		args = append(args, []string{"--find", filePath}...)
+	}
 	_, err = NewLosetupCommand().AddArgs(args...).Run()
 	if err != nil {
 		return nil, err
@@ -166,6 +170,14 @@ func AttachDevice(filePath string, partScan bool) (*Device, error) {
 	return dev, nil
 }
 
+func AttachDeviceWithPath(devPath string, filePath string, partScan bool) (*Device, error) {
+	return attachDevice(devPath, filePath, partScan)
+}
+
+func AttachDevice(filePath string, partScan bool) (*Device, error) {
+	return attachDevice("", filePath, partScan)
+}
+
 // converts a raw key value pair string into a map of key value pairs
 // example raw string of `foo="0" bar="1" baz="biz"` is returned as:
 // map[string]string{"foo":"0", "bar":"1", "baz":"biz"}

pkg/util/losetup/manager/loop.go

@@ -0,0 +1,226 @@
+package manager
+
+import (
+	"fmt"
+	"math/rand"
+	"strconv"
+	"strings"
+	"sync"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/losetup"
+	"yunion.io/x/onecloud/pkg/util/losetup/ioctl"
+	"yunion.io/x/onecloud/pkg/util/mountutils"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type ILoopDevice interface {
+	// GetDevicePath 返回设备路径
+	GetDevicePath() string
+	// IsUsed 检查设备是否在使用中
+	IsUsed() bool
+	// SetUsed 设置设备使用状态
+	SetUsed(used bool)
+}
+
+type ILoopManager interface {
+	AttachDevice(filePath string, partScan bool) (*losetup.Device, error)
+	DetachDevice(devPath string) error
+}
+
+// loopDevice Loop设备实现
+type loopDevice struct {
+	devicePath string
+	used       bool
+	lock       *sync.Mutex
+}
+
+// NewLoopDevice 创建新的Loop设备
+func NewLoopDevice(devicePath string) ILoopDevice {
+	return &loopDevice{
+		devicePath: devicePath,
+		used:       fileutils2.IsBlockDeviceUsed(devicePath),
+		lock:       &sync.Mutex{},
+	}
+}
+
+func (d *loopDevice) GetDevicePath() string {
+	return d.devicePath
+}
+
+func (d *loopDevice) IsUsed() bool {
+	return d.used
+}
+
+func (d *loopDevice) SetUsed(used bool) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	d.used = used
+}
+
+func (d *loopDevice) Detach() error {
+	// 获取所有挂载点
+	mountPoints, err := d.getMountPoints()
+	if err != nil {
+		return errors.Wrap(err, "getMountPoints")
+	}
+
+	// 卸载所有挂载点
+	for _, mountPoint := range mountPoints {
+		if err := mountutils.Unmount(mountPoint, false); err != nil {
+			return errors.Wrapf(err, "umount %s", mountPoint)
+		}
+	}
+
+	// 断开loop设备
+	if err := losetup.DetachDevice(d.GetDevicePath()); err != nil {
+		return errors.Wrapf(err, "detach loop device %s", d.GetDevicePath())
+	}
+
+	return nil
+}
+
+func (d *loopDevice) getMountPoints() ([]string, error) {
+	cmd := fmt.Sprintf("mount | grep %sp1 | awk '{print $3}'", d.GetDevicePath())
+	output, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", cmd).Output()
+	if err != nil {
+		return nil, errors.Wrapf(err, "exec cmd %s: %s", cmd, output)
+	}
+	return strings.Split(string(output), "\n"), nil
+}
+
+func (m *loopManager) initDevices() error {
+	// 使用 ls 和 grep 命令列出所有以数字结尾的 /dev/loop* 设备
+	cmd := "ls /dev/loop* | grep -E 'loop[0-9]+$'"
+	output, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", cmd).Output()
+	if err != nil {
+		return errors.Wrap(err, "list loop devices")
+	}
+
+	// 按行分割输出
+	devices := strings.Split(string(output), "\n")
+	for _, dev := range devices {
+		dev = strings.TrimSpace(dev)
+		if dev == "" {
+			continue
+		}
+		m.devices[dev] = NewLoopDevice(dev)
+	}
+
+	return nil
+}
+
+// loopManager Loop设备管理器
+type loopManager struct {
+	devices    map[string]ILoopDevice
+	actionLock *sync.Mutex
+	mapLock    *sync.Mutex
+}
+
+// NewLoopManager 创建新的Loop管理器
+func newLoopManager() (ILoopManager, error) {
+	ret := &loopManager{
+		devices:    make(map[string]ILoopDevice),
+		actionLock: &sync.Mutex{},
+		mapLock:    new(sync.Mutex),
+	}
+
+	if err := ret.initDevices(); err != nil {
+		return ret, errors.Wrap(err, "initDevices")
+	}
+	return ret, nil
+}
+
+func (m *loopManager) findNewDeviceNumber() (int, error) {
+	// 获取所有已使用的设备号
+	usedNumbers := make(map[int]bool)
+	for name := range m.devices {
+		// 从设备名称中提取数字,例如从 /dev/loop0 中提取 0
+		numStr := strings.TrimPrefix(name, "/dev/loop")
+		num, err := strconv.Atoi(numStr)
+		if err != nil {
+			return -1, errors.Wrapf(err, "parse device number from %s", name)
+		}
+		usedNumbers[num] = true
+	}
+
+	// 从0开始查找第一个未使用的设备号
+	for i := 0; i < 200; i++ {
+		if !usedNumbers[i] {
+			// 创建新的loop设备
+			return i, nil
+		}
+	}
+	return -1, errors.Wrap(errors.ErrNotFound, "No available device found")
+}
+
+func (m *loopManager) AttachDevice(filePath string, partScan bool) (*losetup.Device, error) {
+	m.actionLock.Lock()
+	defer m.actionLock.Unlock()
+
+	dev, err := m.acquireDevice()
+	if err != nil {
+		return nil, errors.Wrap(err, "AcquireDevice")
+	}
+	loDev, err := losetup.AttachDeviceWithPath(dev.GetDevicePath(), filePath, partScan)
+	if err != nil {
+		return nil, errors.Wrapf(err, "AttachDeviceWithPath: %s, filePath: %s", dev.GetDevicePath(), filePath)
+	}
+
+	return loDev, nil
+}
+
+func (m *loopManager) DetachDevice(devPath string) error {
+	m.actionLock.Lock()
+	defer m.actionLock.Unlock()
+
+	if err := losetup.DetachDevice(devPath); err != nil {
+		return errors.Wrapf(err, "DetachDevice: %s", devPath)
+	}
+	m.releaseDevice(devPath)
+	return nil
+}
+
+func (m *loopManager) acquireDevice() (ILoopDevice, error) {
+	m.mapLock.Lock()
+	defer m.mapLock.Unlock()
+
+	freeDevices := make([]ILoopDevice, 0)
+	for _, device := range m.devices {
+		if !device.IsUsed() {
+			freeDevices = append(freeDevices, device)
+		}
+	}
+	if len(freeDevices) > 0 {
+		device := freeDevices[rand.Intn(len(freeDevices))]
+		device.SetUsed(true)
+		return device, nil
+	}
+
+	// create new device
+	num, err := m.findNewDeviceNumber()
+	if err != nil {
+		return nil, errors.Wrap(err, "findNewDeviceNumber")
+	}
+	devPath, err := ioctl.AddDevice(num)
+	if err != nil {
+		return nil, errors.Wrapf(err, "add device %d", num)
+	}
+	device := NewLoopDevice(devPath)
+	m.devices[devPath] = device
+	device.SetUsed(true)
+
+	return device, nil
+}
+
+func (m *loopManager) releaseDevice(devPath string) {
+	m.mapLock.Lock()
+	defer m.mapLock.Unlock()
+
+	if dev, ok := m.devices[devPath]; ok {
+		dev.SetUsed(false)
+	}
+}

pkg/util/losetup/manager/manager.go

@@ -97,11 +97,17 @@ func newDetachOption(devPath string) *execOption {
 
 type manager struct {
 	execOptCh chan *execOption
+	loopMan   ILoopManager
 }
 
 func newManager() iManager {
+	loopMan, err := newLoopManager()
+	if err != nil {
+		log.Fatalf("new loop manager error: %v", err)
+	}
 	m := &manager{
 		execOptCh: make(chan *execOption),
+		loopMan:   loopMan,
 	}
 	go func() {
 		m.startExec()
@@ -132,14 +138,14 @@ func (m *manager) startExec() {
 		switch execOpt.Type {
 		case EXEC_ATTACH:
 			log.Infof("attach %s", jsonutils.Marshal(execOpt.Attach))
-			dev, err := losetup.AttachDevice(execOpt.Attach.FilePath, execOpt.Attach.PartScan)
+			dev, err := m.loopMan.AttachDevice(execOpt.Attach.FilePath, execOpt.Attach.PartScan)
 			execOpt.result <- execResult{
 				attachedDevice: dev,
 				error:          err,
 			}
 		case EXEC_DETACH:
 			log.Infof("detach %s", jsonutils.Marshal(execOpt.Detach))
-			err := losetup.DetachDevice(execOpt.Detach.DevicePath)
+			err := m.loopMan.DetachDevice(execOpt.Detach.DevicePath)
 			execOpt.result <- execResult{
 				error: err,
 			}