fix(host): backup container post overlay (#21773)

pkg/apis/container.go

@@ -172,11 +172,13 @@ var (
 )
 
 type ContainerVolumeMount struct {
-	Type     ContainerVolumeMountType      `json:"type"`
-	Disk     *ContainerVolumeMountDisk     `json:"disk"`
-	HostPath *ContainerVolumeMountHostPath `json:"host_path"`
-	Text     *ContainerVolumeMountText     `json:"text"`
-	CephFS   *ContainerVolumeMountCephFS   `json:"ceph_fs"`
+	// 用于标识当前 pod volume mount 的唯一性
+	UniqueName string                        `json:"unique_name"`
+	Type       ContainerVolumeMountType      `json:"type"`
+	Disk       *ContainerVolumeMountDisk     `json:"disk"`
+	HostPath   *ContainerVolumeMountHostPath `json:"host_path"`
+	Text       *ContainerVolumeMountText     `json:"text"`
+	CephFS     *ContainerVolumeMountCephFS   `json:"ceph_fs"`
 	// Mounted read-only if true, read-write otherwise (false or unspecified).
 	ReadOnly bool `json:"read_only"`
 	// Path within the container at which the volume should be mounted.  Must

pkg/apis/host/container.go

@@ -40,11 +40,13 @@ type ContainerVolumeMountCephFS struct {
 }
 
 type ContainerVolumeMount struct {
-	Type     apis.ContainerVolumeMountType      `json:"type"`
-	Disk     *ContainerVolumeMountDisk          `json:"disk"`
-	HostPath *apis.ContainerVolumeMountHostPath `json:"host_path"`
-	Text     *apis.ContainerVolumeMountText     `json:"text"`
-	CephFS   *ContainerVolumeMountCephFS        `json:"ceph_fs"`
+	// 用于标识当前 pod volume mount 的唯一性
+	UniqueName string                             `json:"unique_name"`
+	Type       apis.ContainerVolumeMountType      `json:"type"`
+	Disk       *ContainerVolumeMountDisk          `json:"disk"`
+	HostPath   *apis.ContainerVolumeMountHostPath `json:"host_path"`
+	Text       *apis.ContainerVolumeMountText     `json:"text"`
+	CephFS     *ContainerVolumeMountCephFS        `json:"ceph_fs"`
 	// Mounted read-only if true, read-write otherwise (false or unspecified).
 	ReadOnly bool `json:"read_only"`
 	// Path within the container at which the volume should be mounted.  Must

pkg/compute/guestdrivers/pod.go

@@ -87,15 +87,26 @@ func (p *SPodDriver) ValidateCreateData(ctx context.Context, userCred mcclient.T
 		return nil, errors.Wrap(err, "validate port mappings")
 	}*/
 
-	sameName := ""
+	ctrNames := sets.NewString()
+	volUniqNames := sets.NewString()
 	for idx, ctr := range input.Pod.Containers {
 		if err := p.validateContainerData(ctx, userCred, idx, input.Name, ctr, input); err != nil {
 			return nil, errors.Wrapf(err, "data of %d container", idx)
 		}
-		if ctr.Name == sameName {
+		if ctrNames.Has(ctr.Name) {
 			return nil, httperrors.NewDuplicateNameError("same name %s of containers", ctr.Name)
 		}
-		sameName = ctr.Name
+		ctrNames.Insert(ctr.Name)
+		for volIdx := range ctr.VolumeMounts {
+			vol := ctr.VolumeMounts[volIdx]
+			if vol.UniqueName != "" {
+				if volUniqNames.Has(vol.UniqueName) {
+					return nil, httperrors.NewDuplicateNameError("same volume unique name %s", fmt.Sprintf("container %s volume_mount %d %s", ctr.Name, volIdx, vol.UniqueName))
+				} else {
+					volUniqNames.Insert(vol.UniqueName)
+				}
+			}
+		}
 	}
 
 	return input, nil

pkg/compute/models/containers.go

@@ -227,7 +227,24 @@ func (m *SContainerManager) ValidateSpecVolumeMounts(ctx context.Context, userCr
 	if err != nil {
 		return errors.Wrap(err, "GetVolumeMountRelations")
 	}
+
+	curCtrs, _ := m.GetContainersByPod(pod.GetId())
+	volUniqNames := sets.NewString()
+	for idx := range curCtrs {
+		for _, vol := range curCtrs[idx].Spec.VolumeMounts {
+			if vol.UniqueName != "" {
+				volUniqNames.Insert(vol.UniqueName)
+			}
+		}
+	}
 	for idx, vm := range spec.VolumeMounts {
+		if vm.UniqueName != "" {
+			if volUniqNames.Has(vm.UniqueName) {
+				return httperrors.NewDuplicateNameError("volume_mount unique_name %s", vm.UniqueName)
+			} else {
+				volUniqNames.Insert(vm.UniqueName)
+			}
+		}
 		newVm, err := m.ValidateSpecVolumeMount(ctx, userCred, pod, vm)
 		if err != nil {
 			return errors.Wrapf(err, "validate volume mount %s", jsonutils.Marshal(vm))
@@ -478,6 +495,7 @@ func (vm *ContainerVolumeMountRelation) toCephFSMount(fs *apis.ContainerVolumeMo
 
 func (vm *ContainerVolumeMountRelation) ToHostMount(ctx context.Context, userCred mcclient.TokenCredential) (*hostapi.ContainerVolumeMount, error) {
 	ret := &hostapi.ContainerVolumeMount{
+		UniqueName:     vm.VolumeMount.UniqueName,
 		Type:           vm.VolumeMount.Type,
 		Disk:           nil,
 		CephFS:         nil,

pkg/hostman/container/volume_mount/disk/disk.go

@@ -42,6 +42,11 @@ type IVolumeMountDisk interface {
 
 	MountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error
 	UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error
+
+	GetHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error)
+
+	GetPostOverlayRootWorkDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error)
+	GetPostOverlayRootUpperDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error)
 }
 
 type disk struct {
@@ -61,7 +66,7 @@ func (d disk) GetType() apis.ContainerVolumeMountType {
 	return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
 }
 
-func (d disk) getHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) {
+func (d disk) GetHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) {
 	diskInput := vm.Disk
 	if diskInput == nil {
 		return "", httperrors.NewNotEmptyError("disk is nil")
@@ -71,7 +76,7 @@ func (d disk) getHostDiskRootPath(pod volume_mount.IPodInfo, vm *hostapi.Contain
 }
 
 func (d disk) getRuntimeMountHostPath(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount) (string, error) {
-	hostPath, err := d.getHostDiskRootPath(pod, vm)
+	hostPath, err := d.GetHostDiskRootPath(pod, vm)
 	if err != nil {
 		return "", errors.Wrap(err, "get host disk root path")
 	}
@@ -254,6 +259,9 @@ func (d disk) Unmount(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.Conta
 			return errors.Wrapf(err, "DisconnectDisk %s %s", iDisk.GetPath(), mntPoint)
 		}
 	}
+	if err := volume_mount.RemoveDir(mntPoint); err != nil {
+		return errors.Wrapf(err, "remove dir %s", mntPoint)
+	}
 	return nil
 }
 

pkg/hostman/container/volume_mount/disk/post_overlay.go

@@ -39,6 +39,26 @@ func (d disk) UnmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *h
 	return d.newPostOverlay().unmountPostOverlays(pod, ctrId, vm, ovs, clearLayers)
 }
 
+func (d disk) getPostOverlayRootPrefixDir(prefixDir string, pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) {
+	hostPath, err := d.GetHostDiskRootPath(pod, vm)
+	if err != nil {
+		return "", errors.Wrap(err, "get host disk root path")
+	}
+	uniqDir := ctrId
+	if vm.UniqueName != "" {
+		uniqDir = vm.UniqueName
+	}
+	return filepath.Join(hostPath, prefixDir, uniqDir), nil
+}
+
+func (d disk) GetPostOverlayRootWorkDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) {
+	return d.getPostOverlayRootPrefixDir(POST_OVERLAY_PREFIX_WORK_DIR, pod, vm, ctrId)
+}
+
+func (d disk) GetPostOverlayRootUpperDir(pod volume_mount.IPodInfo, vm *hostapi.ContainerVolumeMount, ctrId string) (string, error) {
+	return d.getPostOverlayRootPrefixDir(POST_OVERLAY_PREFIX_UPPER_DIR, pod, vm, ctrId)
+}
+
 type iDiskPostOverlay interface {
 	mountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay) error
 	unmountPostOverlays(pod volume_mount.IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount, ovs []*apis.ContainerVolumeMountDiskPostOverlay, clearLayers bool) error
@@ -79,11 +99,12 @@ func (d diskPostOverlay) getPostOverlayDirWithPrefix(
 	ov *apis.ContainerVolumeMountDiskPostOverlay,
 	ensure bool,
 ) (string, error) {
-	hostPath, err := d.disk.getHostDiskRootPath(pod, vm)
+	rootPath, err := d.disk.getPostOverlayRootPrefixDir(prefixDir, pod, vm, ctrId)
 	if err != nil {
-		return "", errors.Wrap(err, "get host disk root path")
+		return "", errors.Wrap(err, "get post overlay root path")
 	}
-	workDir := filepath.Join(hostPath, prefixDir, ctrId, ov.ContainerTargetDir)
+
+	workDir := filepath.Join(rootPath, ov.ContainerTargetDir)
 	if ensure {
 		if err := volume_mount.EnsureDir(workDir); err != nil {
 			return "", errors.Wrapf(err, "make %s", workDir)

pkg/hostman/guestman/pod.go

@@ -62,6 +62,7 @@ import (
 	imagemod "yunion.io/x/onecloud/pkg/mcclient/modules/image"
 	"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/mountutils"
 	"yunion.io/x/onecloud/pkg/util/pod"
 	"yunion.io/x/onecloud/pkg/util/pod/logs"
 	"yunion.io/x/onecloud/pkg/util/pod/nerdctl"
@@ -2155,6 +2156,7 @@ func (s *sPodGuestInstance) DoSnapshot(ctx context.Context, params *SDiskSnapsho
 		return nil, errors.Wrap(err, "EnsureBackupDir")
 	}
 	defer storageman.CleanupDirOrFile(tmpBackRootDir)
+	povTmpBackRootDir := []string{}
 	for _, vol := range vols {
 		drv := volume_mount.GetDriver(vol.Type)
 		if err := drv.Mount(s, input.ContainerId, vol); err != nil {
@@ -2179,27 +2181,51 @@ func (s *sPodGuestInstance) DoSnapshot(ctx context.Context, params *SDiskSnapsho
 				return nil, errors.Wrapf(err, "touch %s: %s", targetBindMntPath, out)
 			}
 		} else {
-			if out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", targetBindMntPath).Output(); err != nil {
-				return nil, errors.Wrapf(err, "mkdir -p %s: %s", targetBindMntPath, out)
+			if err := volume_mount.EnsureDir(targetBindMntPath); err != nil {
+				return nil, errors.Wrap(err, "ensure dir")
 			}
 		}
 		// do bind mount
-		out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", "--bind", mntPath, targetBindMntPath).Output()
-		if err != nil {
-			return nil, errors.Wrapf(err, "bind mount %s to %s: %s", mntPath, targetBindMntPath, out)
+		if err := mountutils.MountBind(mntPath, targetBindMntPath); err != nil {
+			return nil, errors.Wrapf(err, "bind mount %s to %s", mntPath, targetBindMntPath)
+		}
+		// process post overlay
+		diskDrv := drv.(disk.IVolumeMountDisk)
+		for _, pov := range vol.Disk.PostOverlay {
+			// bind mount post overlay dirs to tmpBackRootDir
+			upperDir, err := diskDrv.GetPostOverlayRootUpperDir(s, vol, input.ContainerId)
+			if err != nil {
+				return nil, errors.Wrapf(err, "get post overlay root upper dir: %s", jsonutils.Marshal(pov))
+			}
+			workDir, err := diskDrv.GetPostOverlayRootWorkDir(s, vol, input.ContainerId)
+			if err != nil {
+				return nil, errors.Wrapf(err, "get post overlay root upper dir: %s", jsonutils.Marshal(pov))
+			}
+			hostDiskRootPath, _ := diskDrv.GetHostDiskRootPath(s, vol)
+			for _, srcDir := range []string{upperDir, workDir} {
+				targetSubDir := strings.TrimPrefix(srcDir, hostDiskRootPath)
+				targetPovBindMntPath := filepath.Join(tmpBackRootDir, targetSubDir)
+				if err := mountutils.MountBind(srcDir, targetPovBindMntPath); err != nil {
+					return nil, errors.Wrap(err, "bind mount post overlay dir")
+				}
+				povTmpBackRootDir = append(povTmpBackRootDir, targetPovBindMntPath)
+			}
 		}
 	}
 	deferUmount := func() error {
 		for _, vol := range vols {
 			// unbind mount
-			// umount
+			for _, povPath := range povTmpBackRootDir {
+				if err := mountutils.Unmount(povPath); err != nil {
+					return errors.Wrapf(err, "umount bind point %s", povTmpBackRootDir)
+				}
+			}
 			targetBindMntPath := filepath.Join(tmpBackRootDir, vol.Disk.SubDirectory)
 			if vol.Disk.StorageSizeFile != "" {
 				targetBindMntPath = filepath.Join(tmpBackRootDir, vol.Disk.StorageSizeFile)
 			}
-			out, err := procutils.NewRemoteCommandAsFarAsPossible("umount", targetBindMntPath).Output()
-			if err != nil {
-				return errors.Wrapf(err, "umount bind point %s: %s", targetBindMntPath, out)
+			if err := mountutils.Unmount(targetBindMntPath); err != nil {
+				return errors.Wrapf(err, "umount bind point %s", targetBindMntPath)
 			}
 		}
 		if !isCtrRunning && !s.IsRunning() {

pkg/util/losetup/ioctl/remove.go

@@ -114,7 +114,7 @@ func DetachAndRemoveDevice(devPath string) error {
 	}
 
 	errs := []error{}
-	for i := 1; i <= 3; i++ {
+	for i := 1; i <= 5; i++ {
 		if err := removeDev(); err != nil {
 			err = errors.Wrapf(err, "remove loop device, %d times", i)
 			log.Warningf("remove device %s: %v", devPath, err)

pkg/util/mountutils/mount.go

@@ -67,6 +67,18 @@ func MountOverlay(lowerDir []string, upperDir string, workDir string, mergedDir
 	})
 }
 
+func MountBind(src, target string) error {
+	return mountWrap(target, func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		bindArgs := []string{"--bind", src, target}
+		if out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", bindArgs...).Output(); err != nil {
+			return errors.Wrapf(err, "mount %v: %s", bindArgs, out)
+		}
+		return nil
+	})
+}
+
 func Unmount(mountPoint string) error {
 	err := unmount(mountPoint)
 	errs := []error{}