Merge pull request #19835 from zexi/container-overlay-vol

feat(region,host): support to mount container overlay host directory

pkg/apis/container.go

@@ -121,11 +121,16 @@ type ContainerVolumeMount struct {
 	Propagation ContainerMountPropagation `json:"propagation,omitempty"`
 }
 
+type ContainerVolumeMountDiskOverlay struct {
+	LowerDir []string `json:"lower_dir"`
+}
+
 type ContainerVolumeMountDisk struct {
-	Index           *int   `json:"index,omitempty"`
-	Id              string `json:"id"`
-	SubDirectory    string `json:"sub_directory"`
-	StorageSizeFile string `json:"storage_size_file"`
+	Index           *int                             `json:"index,omitempty"`
+	Id              string                           `json:"id"`
+	SubDirectory    string                           `json:"sub_directory"`
+	StorageSizeFile string                           `json:"storage_size_file"`
+	Overlay         *ContainerVolumeMountDiskOverlay `json:"overlay"`
 }
 
 type ContainerVolumeMountHostPathType string

pkg/compute/container_drivers/volume_mount/disk.go

@@ -75,6 +75,9 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre
 			return nil, httperrors.NewNotFoundError("not found pod disk by %s", disk.Id)
 		}
 	}
+	if err := d.validateOverlay(vm); err != nil {
+		return nil, errors.Wrapf(err, "validate overlay")
+	}
 	return vm, nil
 }
 
@@ -99,3 +102,22 @@ func (d disk) ValidatePodCreateData(ctx context.Context, userCred mcclient.Token
 	}
 	return nil
 }
+
+func (d disk) validateOverlay(vm *apis.ContainerVolumeMount) error {
+	if vm.Disk.Overlay == nil {
+		return nil
+	}
+	ov := vm.Disk.Overlay
+	if len(ov.LowerDir) == 0 {
+		return httperrors.NewNotEmptyError("lower_dir is required")
+	}
+	for idx, ld := range ov.LowerDir {
+		if ld == "" {
+			return httperrors.NewNotEmptyError("empty %d dir", idx)
+		}
+		if ld == "/" {
+			return httperrors.NewInputParameterError("can't use '/' as lower_dir")
+		}
+	}
+	return nil
+}

pkg/hostman/container/volume_mount/disk.go

@@ -3,8 +3,8 @@ package volume_mount
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -30,7 +30,7 @@ func (d disk) GetType() apis.ContainerVolumeMountType {
 	return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK
 }
 
-func (d disk) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) {
+func (d disk) getRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) {
 	diskInput := vm.Disk
 	if diskInput == nil {
 		return "", httperrors.NewNotEmptyError("disk is nil")
@@ -45,6 +45,18 @@ func (d disk) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMoun
 	return hostPath, nil
 }
 
+func (d disk) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) (string, error) {
+	hostPath, err := d.getRuntimeMountHostPath(pod, vm)
+	if err != nil {
+		return "", errors.Wrap(err, "get runtime mount host_path")
+	}
+	overlay := vm.Disk.Overlay
+	if overlay == nil {
+		return hostPath, nil
+	}
+	return d.getOverlayMergedDir(pod, ctrId, vm, hostPath), nil
+}
+
 func (d disk) getPodDisk(pod IPodInfo, vm *apis.ContainerVolumeMount) (storageman.IDisk, *desc.SGuestDisk, error) {
 	var disk *desc.SGuestDisk = nil
 	disks := pod.GetDisks()
@@ -82,7 +94,19 @@ func (d disk) getDiskStorageDriver(pod IPodInfo, vm *apis.ContainerVolumeMount)
 	return drv, nil
 }
 
-func (d disk) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
+func (d disk) getOverlayDir(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount, upperDir string, suffix string) string {
+	return filepath.Join(pod.GetVolumesOverlayDir(), vm.Disk.Id, ctrId, fmt.Sprintf("%s-%s", filepath.Base(upperDir), suffix))
+}
+
+func (d disk) getOverlayWorkDir(upperDir string) string {
+	return fmt.Sprintf("%s-work", upperDir)
+}
+
+func (d disk) getOverlayMergedDir(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount, upperDir string) string {
+	return d.getOverlayDir(pod, ctrId, vm, upperDir, "merged")
+}
+
+func (d disk) Mount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error {
 	iDisk, gd, err := d.getPodDisk(pod, vm)
 	if err != nil {
 		return errors.Wrap(err, "get pod disk interface")
@@ -95,7 +119,6 @@ func (d disk) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
 	if err != nil {
 		return errors.Wrapf(err, "CheckConnect %s", iDisk.GetPath())
 	}
-	log.Infof("=======check connect: %q %q %v", iDisk.GetPath(), devPath, isConnected)
 	if !isConnected {
 		devPath, err = drv.ConnectDisk(iDisk.GetPath())
 		if err != nil {
@@ -118,6 +141,11 @@ func (d disk) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
 			return errors.Wrapf(err, "create storage file %s inside %s", vmDisk.StorageSizeFile, mntPoint)
 		}
 	}
+	if vmDisk.Overlay != nil {
+		if err := d.mountOverlay(pod, ctrId, vm); err != nil {
+			return errors.Wrapf(err, "mount container %s overlay dir: %#v", ctrId, vmDisk.Overlay)
+		}
+	}
 	return nil
 }
 
@@ -136,7 +164,7 @@ func (d disk) createStorageSizeFile(iDisk storageman.IDisk, mntPoint string, inp
 	return nil
 }
 
-func (d disk) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
+func (d disk) Unmount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error {
 	iDisk, _, err := d.getPodDisk(pod, vm)
 	if err != nil {
 		return errors.Wrap(err, "get pod disk interface")
@@ -145,6 +173,11 @@ func (d disk) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
 	if err != nil {
 		return errors.Wrap(err, "get disk storage driver")
 	}
+	if vm.Disk.Overlay != nil {
+		if err := d.unmoutOverlay(pod, ctrId, vm); err != nil {
+			return errors.Wrapf(err, "umount overlay")
+		}
+	}
 	mntPoint := pod.GetDiskMountPoint(iDisk)
 	if err := container_storage.Unmount(mntPoint); err != nil {
 		return errors.Wrapf(err, "unmount %s", mntPoint)
@@ -160,3 +193,36 @@ func (d disk) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
 	}
 	return nil
 }
+
+func (d disk) unmoutOverlay(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error {
+	upperDir, err := d.getRuntimeMountHostPath(pod, vm)
+	if err != nil {
+		return errors.Wrap(err, "getRuntimeMountHostPath")
+	}
+	overlayDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir)
+	return container_storage.Unmount(overlayDir)
+}
+
+func (d disk) mountOverlay(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error {
+	vmDisk := vm.Disk
+	lowerDir := vmDisk.Overlay.LowerDir
+	upperDir, err := d.getRuntimeMountHostPath(pod, vm)
+	if err != nil {
+		return errors.Wrap(err, "getRuntimeMountHostPath")
+	}
+	workDir := d.getOverlayWorkDir(upperDir)
+	mergedDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir)
+	for _, dir := range []string{workDir, mergedDir} {
+		out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output()
+		if err != nil {
+			return errors.Wrapf(err, "make directory %s: %s", dir, out)
+		}
+	}
+
+	overlayArgs := []string{"-t", "overlay", "overlay", "-o", fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(lowerDir, ":"), upperDir, workDir), mergedDir}
+	if out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", overlayArgs...).Output(); err != nil {
+		return errors.Wrapf(err, "mount %v: %s", overlayArgs, out)
+	}
+
+	return nil
+}

pkg/hostman/container/volume_mount/host_path.go

@@ -14,11 +14,11 @@ func init() {
 
 type hostLocal struct{}
 
-func (h hostLocal) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
+func (h hostLocal) Mount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error {
 	return nil
 }
 
-func (h hostLocal) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error {
+func (h hostLocal) Unmount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error {
 	return nil
 }
 
@@ -30,7 +30,7 @@ func (h hostLocal) GetType() apis.ContainerVolumeMountType {
 	return apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH
 }
 
-func (h hostLocal) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) {
+func (h hostLocal) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) (string, error) {
 	host := vm.HostPath
 	if host == nil {
 		return "", httperrors.NewNotEmptyError("host_local is nil")

pkg/hostman/container/volume_mount/interface.go

@@ -28,15 +28,16 @@ func GetDriver(typ apis.ContainerVolumeMountType) IVolumeMount {
 
 type IPodInfo interface {
 	GetVolumesDir() string
+	GetVolumesOverlayDir() string
 	GetDisks() []*desc.SGuestDisk
 	GetDiskMountPoint(disk storageman.IDisk) string
 }
 
 type IVolumeMount interface {
 	GetType() apis.ContainerVolumeMountType
-	GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error)
-	Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error
-	Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error
+	GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) (string, error)
+	Mount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error
+	Unmount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error
 }
 
 func GetRuntimeVolumeMountPropagation(input apis.ContainerMountPropagation) runtimeapi.MountPropagation {

pkg/hostman/guestman/pod.go

@@ -216,38 +216,51 @@ func (s *sPodGuestInstance) GetDisks() []*desc.SGuestDisk {
 }
 
 func (s *sPodGuestInstance) mountPodVolumes() error {
-	for _, vol := range s.getContainerVolumeMounts() {
-		if err := volume_mount.GetDriver(vol.Type).Mount(s, vol); err != nil {
-			return errors.Wrapf(err, "mount volume %s", jsonutils.Marshal(vol))
+	for ctrId, vols := range s.getContainerVolumeMounts() {
+		for _, vol := range vols {
+			if err := volume_mount.GetDriver(vol.Type).Mount(s, ctrId, vol); err != nil {
+				return errors.Wrapf(err, "mount volume %s, ctrId %s", jsonutils.Marshal(vol), ctrId)
+			}
 		}
 	}
 	return nil
 }
 
 func (s *sPodGuestInstance) umountPodVolumes() error {
-	for _, vol := range s.getContainerVolumeMounts() {
-		if err := volume_mount.GetDriver(vol.Type).Unmount(s, vol); err != nil {
-			return errors.Wrapf(err, "Unmount volume %s", jsonutils.Marshal(vol))
+	for ctrId, vols := range s.getContainerVolumeMounts() {
+		for _, vol := range vols {
+			if err := volume_mount.GetDriver(vol.Type).Unmount(s, ctrId, vol); err != nil {
+				return errors.Wrapf(err, "Unmount volume %s, ctrId %s", jsonutils.Marshal(vol), ctrId)
+			}
 		}
 	}
 	return nil
 }
 
-func (s *sPodGuestInstance) getContainerVolumeMounts() []*apis.ContainerVolumeMount {
-	mnts := make([]*apis.ContainerVolumeMount, 0)
+func (s *sPodGuestInstance) getContainerVolumeMounts() map[string][]*apis.ContainerVolumeMount {
+	result := make(map[string][]*apis.ContainerVolumeMount, 0)
 	for _, ctr := range s.GetDesc().Containers {
+		mnts, ok := result[ctr.Id]
+		if !ok {
+			mnts = make([]*apis.ContainerVolumeMount, 0)
+		}
 		for _, vol := range ctr.Spec.VolumeMounts {
 			tmp := vol
 			mnts = append(mnts, tmp)
 		}
+		result[ctr.Id] = mnts
 	}
-	return mnts
+	return result
 }
 
 func (s *sPodGuestInstance) GetVolumesDir() string {
 	return filepath.Join(s.HomeDir(), "volumes")
 }
 
+func (s *sPodGuestInstance) GetVolumesOverlayDir() string {
+	return filepath.Join(s.GetVolumesDir(), "overlay")
+}
+
 func (s *sPodGuestInstance) GetDiskMountPoint(disk storageman.IDisk) string {
 	return filepath.Join(s.GetVolumesDir(), disk.GetId())
 }
@@ -762,7 +775,7 @@ func (s *sPodGuestInstance) getLxcfsMounts() []*runtimeapi.Mount {
 	}
 }
 
-func (s *sPodGuestInstance) getContainerMounts(input *hostapi.ContainerCreateInput) ([]*runtimeapi.Mount, error) {
+func (s *sPodGuestInstance) getContainerMounts(ctrId string, input *hostapi.ContainerCreateInput) ([]*runtimeapi.Mount, error) {
 	inputMounts := input.Spec.VolumeMounts
 	if len(inputMounts) == 0 {
 		return make([]*runtimeapi.Mount, 0), nil
@@ -776,7 +789,7 @@ func (s *sPodGuestInstance) getContainerMounts(input *hostapi.ContainerCreateInp
 			SelinuxRelabel: im.SelinuxRelabel,
 			Propagation:    volume_mount.GetRuntimeVolumeMountPropagation(im.Propagation),
 		}
-		hostPath, err := volume_mount.GetDriver(im.Type).GetRuntimeMountHostPath(s, im)
+		hostPath, err := volume_mount.GetDriver(im.Type).GetRuntimeMountHostPath(s, ctrId, im)
 		if err != nil {
 			return nil, errors.Wrapf(err, "get runtime host mount path of %s", jsonutils.Marshal(im))
 		}
@@ -803,7 +816,7 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie
 	if err != nil {
 		return "", errors.Wrap(err, "getPodSandboxConfig")
 	}
-	mounts, err := s.getContainerMounts(input)
+	mounts, err := s.getContainerMounts(ctrId, input)
 	if err != nil {
 		return "", errors.Wrap(err, "get container mounts")
 	}

pkg/mcclient/options/compute/containers.go

@@ -199,6 +199,13 @@ func parseContainerVolumeMount(vmStr string) (*apis.ContainerVolumeMount, error)
 				vm.Disk = &apis.ContainerVolumeMountDisk{}
 			}
 			vm.Disk.StorageSizeFile = val
+		case "overlay":
+			if vm.Disk == nil {
+				vm.Disk = &apis.ContainerVolumeMountDisk{}
+			}
+			vm.Disk.Overlay = &apis.ContainerVolumeMountDiskOverlay{
+				LowerDir: strings.Split(val, ":"),
+			}
 		}
 	}
 	return vm, nil

pkg/mcclient/options/compute/containers_test.go

@@ -29,12 +29,17 @@ func Test_parseContainerVolumeMount(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			args: "readonly=true,mount_path=/data,disk_index=0",
+			args: "readonly=true,mount_path=/data,disk_index=0,overlay=/abc:/bca",
 			want: &apis.ContainerVolumeMount{
 				ReadOnly:  true,
 				MountPath: "/data",
-				Disk:      &apis.ContainerVolumeMountDisk{Index: &index0},
-				Type:      apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
+				Disk: &apis.ContainerVolumeMountDisk{
+					Index: &index0,
+					Overlay: &apis.ContainerVolumeMountDiskOverlay{
+						LowerDir: []string{"/abc", "/bca"},
+					},
+				},
+				Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
 			},
 		},
 		{