From e30d669d19afa700a1160eb06b5e7444a481c540 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Mon, 25 Aug 2025 11:48:01 +0800 Subject: [PATCH] feat(host): support single file to use post overlay (#23159) --- .../volume_mount/disk/post_overlay.go | 29 +++- .../disk/post_overlay_hostpath.go | 137 +++++++++++++++++- pkg/hostman/container/volume_mount/helper.go | 18 ++- 3 files changed, 173 insertions(+), 11 deletions(-) diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay.go b/pkg/hostman/container/volume_mount/disk/post_overlay.go index fb552018b0..b07e2f7aa6 100644 --- a/pkg/hostman/container/volume_mount/disk/post_overlay.go +++ b/pkg/hostman/container/volume_mount/disk/post_overlay.go @@ -26,8 +26,10 @@ import ( ) const ( - POST_OVERLAY_PREFIX_WORK_DIR = "_post_overlay_work_" - POST_OVERLAY_PREFIX_UPPER_DIR = "_post_overlay_upper_" + POST_OVERLAY_PREFIX_LOWER_DIR = "_post_overlay_lower_" + POST_OVERLAY_PREFIX_WORK_DIR = "_post_overlay_work_" + POST_OVERLAY_PREFIX_UPPER_DIR = "_post_overlay_upper_" + POST_OVERLAY_PREFIX_MERGED_DIR = "_post_overlay_merged_" ) func (d disk) MountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error { @@ -128,7 +130,7 @@ func (d diskPostOverlay) getPostOverlayDirWithPrefix( workDir := filepath.Join(rootPath, ov.ContainerTargetDir) if ensure { if err := volume_mount.EnsureDir(workDir); err != nil { - return "", errors.Wrapf(err, "make %s", workDir) + return "", errors.Wrapf(err, "makeDir %s", workDir) } } return workDir, nil @@ -152,7 +154,20 @@ func (d diskPostOverlay) getPostOverlayUpperDir( return d.getPostOverlayDirWithPrefix(POST_OVERLAY_PREFIX_UPPER_DIR, pod, ctrId, vm, ov, ensure) } -func (d diskPostOverlay) getPostOverlayMountpoint(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) (string, error) { +func (d diskPostOverlay) getPostOverlayLowerDir( + pod volume_mount.IPodInfo, ctrId string, + vm *hostapi.ContainerVolumeMount, + ov *apis.ContainerVolumeMountDiskPostOverlay, + ensure bool, +) (string, error) { + return d.getPostOverlayDirWithPrefix(POST_OVERLAY_PREFIX_LOWER_DIR, pod, ctrId, vm, ov, ensure) +} + +func (d diskPostOverlay) getPostOverlayMergedDir(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, ensure bool) (string, error) { + return d.getPostOverlayDirWithPrefix(POST_OVERLAY_PREFIX_MERGED_DIR, pod, ctrId, vm, ov, ensure) +} + +func (d diskPostOverlay) getPostOverlayMountpoint(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, ensure bool) (string, error) { ctrMountHostPath, err := d.disk.GetRuntimeMountHostPath(pod, ctrId, vm) if err != nil { return "", errors.Wrap(err, "get runtime mount host_path") @@ -160,8 +175,10 @@ func (d diskPostOverlay) getPostOverlayMountpoint(pod volume_mount.IPodInfo, ctr // remove hostPath sub_directory path ctrMountHostPath = strings.TrimSuffix(ctrMountHostPath, vm.Disk.SubDirectory) mergedDir := filepath.Join(ctrMountHostPath, ov.ContainerTargetDir) - if err := volume_mount.EnsureDir(mergedDir); err != nil { - return "", errors.Wrap(err, "make merged mountpoint dir") + if ensure { + if err := volume_mount.EnsureDir(mergedDir); err != nil { + return "", errors.Wrap(err, "make merged mountpoint dir") + } } return mergedDir, nil } diff --git a/pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go b/pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go index 596de73ab5..33faf445f8 100644 --- a/pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go +++ b/pkg/hostman/container/volume_mount/disk/post_overlay_hostpath.go @@ -15,11 +15,14 @@ package disk import ( + "path/filepath" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/apis" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/container/volume_mount" + fileutils "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/mountutils" ) @@ -38,7 +41,14 @@ func (p postOverlayHostPath) GetType() apis.ContainerVolumeMountDiskPostOverlayT return apis.CONTAINER_VOLUME_MOUNT_DISK_POST_OVERLAY_HOSTPATH } -func (p postOverlayHostPath) Mount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { +func (p postOverlayHostPath) getSingleFilePath(ov *apis.ContainerVolumeMountDiskPostOverlay) string { + if len(ov.HostLowerDir) == 1 && fileutils.IsFile(ov.HostLowerDir[0]) { + return ov.HostLowerDir[0] + } + return "" +} + +func (p postOverlayHostPath) mountDir(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, true) if err != nil { return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) @@ -49,7 +59,7 @@ func (p postOverlayHostPath) Mount(d diskPostOverlay, pod volume_mount.IPodInfo, return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) } - mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov, true) if err != nil { return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) } @@ -65,8 +75,72 @@ func (p postOverlayHostPath) Mount(d diskPostOverlay, pod volume_mount.IPodInfo, return nil } -func (p postOverlayHostPath) Unmount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, cleanLayers bool) error { - mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov) +func (p postOverlayHostPath) getSingleFileMergedFilePath(mergedDir string, singleFilePath string) string { + return filepath.Join(mergedDir, filepath.Base(singleFilePath)) +} + +func (p postOverlayHostPath) mountSingleFile(singleFilePath string, d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + // 如果是单文件，创建一个目录，再把该文件拷贝过去 + lowerDir, err := d.getPostOverlayLowerDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get single file lower dir for container %s", ctrId) + } + if err := volume_mount.CopyFile(singleFilePath, lowerDir); err != nil { + return errors.Wrapf(err, "copy file %s to %s", singleFilePath, lowerDir) + } + + // 把但文件的 lowerDir 挂载到 targetMergedDir，然后在把 singleFileMergedFilePath bind mount 到 mergedDir(mergedDir 其实是个文件路径) + upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) + } + workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) + } + mergedDst, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + targetMergedDir, err := d.getPostOverlayMergedDir(pod, ctrId, vm, ov, true) + if err != nil { + return errors.Wrapf(err, "get post overlay merged dir for container %s", ctrId) + } + if err := mountutils.MountOverlayWithFeatures([]string{lowerDir}, upperDir, workDir, targetMergedDir, &mountutils.MountOverlayFeatures{ + MetaCopy: true, + }); err != nil { + return errors.Wrapf(err, "mount overlay dir for container %s", ctrId) + } + singleFileMergedFilePath := p.getSingleFileMergedFilePath(targetMergedDir, singleFilePath) + if err := volume_mount.ChangeDirOwnerDirectly(singleFileMergedFilePath, ov.FsUser, ov.FsGroup); err != nil { + return errors.Wrapf(err, "change file %s owner", singleFilePath) + } + if !fileutils.Exists(mergedDst) { + if err := volume_mount.TouchFile(mergedDst); err != nil { + return errors.Wrapf(err, "touch file %s", mergedDst) + } + } + if err := mountutils.MountBind(singleFileMergedFilePath, mergedDst); err != nil { + return errors.Wrapf(err, "bind mount %s to %s", singleFileMergedFilePath, mergedDst) + } + return nil +} + +func (p postOverlayHostPath) Mount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay) error { + // 支持单文件挂载 + singleFilePath := "" + if len(ov.HostLowerDir) == 1 && fileutils.IsFile(ov.HostLowerDir[0]) { + singleFilePath = ov.HostLowerDir[0] + } + if singleFilePath != "" { + return p.mountSingleFile(singleFilePath, d, pod, ctrId, vm, ov) + } else { + return p.mountDir(d, pod, ctrId, vm, ov) + } +} + +func (p postOverlayHostPath) unmountDir(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, cleanLayers bool) error { + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov, false) if err != nil { return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) } @@ -92,3 +166,58 @@ func (p postOverlayHostPath) Unmount(d diskPostOverlay, pod volume_mount.IPodInf } return nil } + +func (p postOverlayHostPath) unmountSingleFile(singleFilePath string, d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, cleanLayers bool) error { + mergedDir, err := d.getPostOverlayMountpoint(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay mountpoint for container %s", ctrId) + } + targetMergedDir, err := d.getPostOverlayMergedDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay merged dir for container %s", ctrId) + } + singleFileMergedFilePath := p.getSingleFileMergedFilePath(targetMergedDir, singleFilePath) + + // 先 unbind mergedDir + if err := mountutils.Unmount(mergedDir, false); err != nil { + return errors.Wrapf(err, "unmount %s of single file %s", mergedDir, singleFileMergedFilePath) + } + + if err := mountutils.Unmount(targetMergedDir, useLazy); err != nil { + return errors.Wrapf(err, "unmount %s", targetMergedDir) + } + if cleanLayers { + upperDir, err := d.getPostOverlayUpperDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay upper dir for container %s", ctrId) + } + workDir, err := d.getPostOverlayWorkDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay work dir for container %s", ctrId) + } + lowerDir, err := d.getPostOverlayLowerDir(pod, ctrId, vm, ov, false) + if err != nil { + return errors.Wrapf(err, "get post overlay lower dir for container %s", ctrId) + } + + for _, dir := range []string{upperDir, workDir, lowerDir, targetMergedDir} { + if err := volume_mount.RemoveDir(dir); err != nil { + return errors.Wrapf(err, "remove dir %s", dir) + } + } + } + return nil +} + +func (p postOverlayHostPath) Unmount(d diskPostOverlay, pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ov *apis.ContainerVolumeMountDiskPostOverlay, useLazy bool, cleanLayers bool) error { + // 支持单文件卸载 + singleFilePath := "" + if len(ov.HostLowerDir) == 1 && fileutils.IsFile(ov.HostLowerDir[0]) { + singleFilePath = ov.HostLowerDir[0] + } + if singleFilePath != "" { + return p.unmountSingleFile(singleFilePath, d, pod, ctrId, vm, ov, useLazy, cleanLayers) + } else { + return p.unmountDir(d, pod, ctrId, vm, ov, useLazy, cleanLayers) + } +} diff --git a/pkg/hostman/container/volume_mount/helper.go b/pkg/hostman/container/volume_mount/helper.go index 639da6feb1..f752fd969a 100644 --- a/pkg/hostman/container/volume_mount/helper.go +++ b/pkg/hostman/container/volume_mount/helper.go @@ -23,6 +23,14 @@ import ( "yunion.io/x/onecloud/pkg/util/procutils" ) +func TouchFile(file string) error { + out, err := procutils.NewRemoteCommandAsFarAsPossible("touch", file).Output() + if err != nil { + return errors.Wrapf(err, "touch %s: %s", file, out) + } + return nil +} + func EnsureDir(dir string) error { out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() if err != nil { @@ -63,7 +71,15 @@ func ChangeDirOwnerDirectly(hostPath string, fsUser, fsGroup *int64) error { } out, err := procutils.NewRemoteCommandAsFarAsPossible("chown", args, hostPath).Output() if err != nil { - return errors.Wrapf(err, "chown -R %s %s: %s", args, hostPath, string(out)) + return errors.Wrapf(err, "chown %s %s: %s", args, hostPath, string(out)) + } + return nil +} + +func CopyFile(src, dst string) error { + out, err := procutils.NewRemoteCommandAsFarAsPossible("cp", src, dst).Output() + if err != nil { + return errors.Wrapf(err, "cp %s %s: %s", src, dst, string(out)) } return nil }