diff --git a/pkg/apis/container.go b/pkg/apis/container.go index a1735fbfc6..2149573449 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -121,11 +121,16 @@ type ContainerVolumeMount struct { Propagation ContainerMountPropagation `json:"propagation,omitempty"` } +type ContainerVolumeMountDiskOverlay struct { + LowerDir []string `json:"lower_dir"` +} + type ContainerVolumeMountDisk struct { - Index *int `json:"index,omitempty"` - Id string `json:"id"` - SubDirectory string `json:"sub_directory"` - StorageSizeFile string `json:"storage_size_file"` + Index *int `json:"index,omitempty"` + Id string `json:"id"` + SubDirectory string `json:"sub_directory"` + StorageSizeFile string `json:"storage_size_file"` + Overlay *ContainerVolumeMountDiskOverlay `json:"overlay"` } type ContainerVolumeMountHostPathType string diff --git a/pkg/compute/container_drivers/volume_mount/disk.go b/pkg/compute/container_drivers/volume_mount/disk.go index e83f491919..bd63cbaef6 100644 --- a/pkg/compute/container_drivers/volume_mount/disk.go +++ b/pkg/compute/container_drivers/volume_mount/disk.go @@ -75,6 +75,9 @@ func (d disk) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCre return nil, httperrors.NewNotFoundError("not found pod disk by %s", disk.Id) } } + if err := d.validateOverlay(vm); err != nil { + return nil, errors.Wrapf(err, "validate overlay") + } return vm, nil } @@ -99,3 +102,22 @@ func (d disk) ValidatePodCreateData(ctx context.Context, userCred mcclient.Token } return nil } + +func (d disk) validateOverlay(vm *apis.ContainerVolumeMount) error { + if vm.Disk.Overlay == nil { + return nil + } + ov := vm.Disk.Overlay + if len(ov.LowerDir) == 0 { + return httperrors.NewNotEmptyError("lower_dir is required") + } + for idx, ld := range ov.LowerDir { + if ld == "" { + return httperrors.NewNotEmptyError("empty %d dir", idx) + } + if ld == "/" { + return httperrors.NewInputParameterError("can't use '/' as lower_dir") + } + } + return nil +} diff --git a/pkg/hostman/container/volume_mount/disk.go b/pkg/hostman/container/volume_mount/disk.go index 80b69ca33c..bac893cc49 100644 --- a/pkg/hostman/container/volume_mount/disk.go +++ b/pkg/hostman/container/volume_mount/disk.go @@ -3,8 +3,8 @@ package volume_mount import ( "fmt" "path/filepath" + "strings" - "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/apis" @@ -30,7 +30,7 @@ func (d disk) GetType() apis.ContainerVolumeMountType { return apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK } -func (d disk) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) { +func (d disk) getRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) { diskInput := vm.Disk if diskInput == nil { return "", httperrors.NewNotEmptyError("disk is nil") @@ -45,6 +45,18 @@ func (d disk) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMoun return hostPath, nil } +func (d disk) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) (string, error) { + hostPath, err := d.getRuntimeMountHostPath(pod, vm) + if err != nil { + return "", errors.Wrap(err, "get runtime mount host_path") + } + overlay := vm.Disk.Overlay + if overlay == nil { + return hostPath, nil + } + return d.getOverlayMergedDir(pod, ctrId, vm, hostPath), nil +} + func (d disk) getPodDisk(pod IPodInfo, vm *apis.ContainerVolumeMount) (storageman.IDisk, *desc.SGuestDisk, error) { var disk *desc.SGuestDisk = nil disks := pod.GetDisks() @@ -82,7 +94,19 @@ func (d disk) getDiskStorageDriver(pod IPodInfo, vm *apis.ContainerVolumeMount) return drv, nil } -func (d disk) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { +func (d disk) getOverlayDir(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount, upperDir string, suffix string) string { + return filepath.Join(pod.GetVolumesOverlayDir(), vm.Disk.Id, ctrId, fmt.Sprintf("%s-%s", filepath.Base(upperDir), suffix)) +} + +func (d disk) getOverlayWorkDir(upperDir string) string { + return fmt.Sprintf("%s-work", upperDir) +} + +func (d disk) getOverlayMergedDir(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount, upperDir string) string { + return d.getOverlayDir(pod, ctrId, vm, upperDir, "merged") +} + +func (d disk) Mount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error { iDisk, gd, err := d.getPodDisk(pod, vm) if err != nil { return errors.Wrap(err, "get pod disk interface") @@ -95,7 +119,6 @@ func (d disk) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { if err != nil { return errors.Wrapf(err, "CheckConnect %s", iDisk.GetPath()) } - log.Infof("=======check connect: %q %q %v", iDisk.GetPath(), devPath, isConnected) if !isConnected { devPath, err = drv.ConnectDisk(iDisk.GetPath()) if err != nil { @@ -118,6 +141,11 @@ func (d disk) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { return errors.Wrapf(err, "create storage file %s inside %s", vmDisk.StorageSizeFile, mntPoint) } } + if vmDisk.Overlay != nil { + if err := d.mountOverlay(pod, ctrId, vm); err != nil { + return errors.Wrapf(err, "mount container %s overlay dir: %#v", ctrId, vmDisk.Overlay) + } + } return nil } @@ -136,7 +164,7 @@ func (d disk) createStorageSizeFile(iDisk storageman.IDisk, mntPoint string, inp return nil } -func (d disk) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { +func (d disk) Unmount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error { iDisk, _, err := d.getPodDisk(pod, vm) if err != nil { return errors.Wrap(err, "get pod disk interface") @@ -145,6 +173,11 @@ func (d disk) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { if err != nil { return errors.Wrap(err, "get disk storage driver") } + if vm.Disk.Overlay != nil { + if err := d.unmoutOverlay(pod, ctrId, vm); err != nil { + return errors.Wrapf(err, "umount overlay") + } + } mntPoint := pod.GetDiskMountPoint(iDisk) if err := container_storage.Unmount(mntPoint); err != nil { return errors.Wrapf(err, "unmount %s", mntPoint) @@ -160,3 +193,36 @@ func (d disk) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { } return nil } + +func (d disk) unmoutOverlay(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error { + upperDir, err := d.getRuntimeMountHostPath(pod, vm) + if err != nil { + return errors.Wrap(err, "getRuntimeMountHostPath") + } + overlayDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) + return container_storage.Unmount(overlayDir) +} + +func (d disk) mountOverlay(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error { + vmDisk := vm.Disk + lowerDir := vmDisk.Overlay.LowerDir + upperDir, err := d.getRuntimeMountHostPath(pod, vm) + if err != nil { + return errors.Wrap(err, "getRuntimeMountHostPath") + } + workDir := d.getOverlayWorkDir(upperDir) + mergedDir := d.getOverlayMergedDir(pod, ctrId, vm, upperDir) + for _, dir := range []string{workDir, mergedDir} { + out, err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + if err != nil { + return errors.Wrapf(err, "make directory %s: %s", dir, out) + } + } + + overlayArgs := []string{"-t", "overlay", "overlay", "-o", fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(lowerDir, ":"), upperDir, workDir), mergedDir} + if out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", overlayArgs...).Output(); err != nil { + return errors.Wrapf(err, "mount %v: %s", overlayArgs, out) + } + + return nil +} diff --git a/pkg/hostman/container/volume_mount/host_path.go b/pkg/hostman/container/volume_mount/host_path.go index 46994d303a..62acaa8f8a 100644 --- a/pkg/hostman/container/volume_mount/host_path.go +++ b/pkg/hostman/container/volume_mount/host_path.go @@ -14,11 +14,11 @@ func init() { type hostLocal struct{} -func (h hostLocal) Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { +func (h hostLocal) Mount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error { return nil } -func (h hostLocal) Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error { +func (h hostLocal) Unmount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error { return nil } @@ -30,7 +30,7 @@ func (h hostLocal) GetType() apis.ContainerVolumeMountType { return apis.CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH } -func (h hostLocal) GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) { +func (h hostLocal) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) (string, error) { host := vm.HostPath if host == nil { return "", httperrors.NewNotEmptyError("host_local is nil") diff --git a/pkg/hostman/container/volume_mount/interface.go b/pkg/hostman/container/volume_mount/interface.go index 0cb6e1d298..c2f837ac20 100644 --- a/pkg/hostman/container/volume_mount/interface.go +++ b/pkg/hostman/container/volume_mount/interface.go @@ -28,15 +28,16 @@ func GetDriver(typ apis.ContainerVolumeMountType) IVolumeMount { type IPodInfo interface { GetVolumesDir() string + GetVolumesOverlayDir() string GetDisks() []*desc.SGuestDisk GetDiskMountPoint(disk storageman.IDisk) string } type IVolumeMount interface { GetType() apis.ContainerVolumeMountType - GetRuntimeMountHostPath(pod IPodInfo, vm *apis.ContainerVolumeMount) (string, error) - Mount(pod IPodInfo, vm *apis.ContainerVolumeMount) error - Unmount(pod IPodInfo, vm *apis.ContainerVolumeMount) error + GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) (string, error) + Mount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error + Unmount(pod IPodInfo, ctrId string, vm *apis.ContainerVolumeMount) error } func GetRuntimeVolumeMountPropagation(input apis.ContainerMountPropagation) runtimeapi.MountPropagation { diff --git a/pkg/hostman/guestman/pod.go b/pkg/hostman/guestman/pod.go index 1b7ff3261d..249692192f 100644 --- a/pkg/hostman/guestman/pod.go +++ b/pkg/hostman/guestman/pod.go @@ -216,38 +216,51 @@ func (s *sPodGuestInstance) GetDisks() []*desc.SGuestDisk { } func (s *sPodGuestInstance) mountPodVolumes() error { - for _, vol := range s.getContainerVolumeMounts() { - if err := volume_mount.GetDriver(vol.Type).Mount(s, vol); err != nil { - return errors.Wrapf(err, "mount volume %s", jsonutils.Marshal(vol)) + for ctrId, vols := range s.getContainerVolumeMounts() { + for _, vol := range vols { + if err := volume_mount.GetDriver(vol.Type).Mount(s, ctrId, vol); err != nil { + return errors.Wrapf(err, "mount volume %s, ctrId %s", jsonutils.Marshal(vol), ctrId) + } } } return nil } func (s *sPodGuestInstance) umountPodVolumes() error { - for _, vol := range s.getContainerVolumeMounts() { - if err := volume_mount.GetDriver(vol.Type).Unmount(s, vol); err != nil { - return errors.Wrapf(err, "Unmount volume %s", jsonutils.Marshal(vol)) + for ctrId, vols := range s.getContainerVolumeMounts() { + for _, vol := range vols { + if err := volume_mount.GetDriver(vol.Type).Unmount(s, ctrId, vol); err != nil { + return errors.Wrapf(err, "Unmount volume %s, ctrId %s", jsonutils.Marshal(vol), ctrId) + } } } return nil } -func (s *sPodGuestInstance) getContainerVolumeMounts() []*apis.ContainerVolumeMount { - mnts := make([]*apis.ContainerVolumeMount, 0) +func (s *sPodGuestInstance) getContainerVolumeMounts() map[string][]*apis.ContainerVolumeMount { + result := make(map[string][]*apis.ContainerVolumeMount, 0) for _, ctr := range s.GetDesc().Containers { + mnts, ok := result[ctr.Id] + if !ok { + mnts = make([]*apis.ContainerVolumeMount, 0) + } for _, vol := range ctr.Spec.VolumeMounts { tmp := vol mnts = append(mnts, tmp) } + result[ctr.Id] = mnts } - return mnts + return result } func (s *sPodGuestInstance) GetVolumesDir() string { return filepath.Join(s.HomeDir(), "volumes") } +func (s *sPodGuestInstance) GetVolumesOverlayDir() string { + return filepath.Join(s.GetVolumesDir(), "overlay") +} + func (s *sPodGuestInstance) GetDiskMountPoint(disk storageman.IDisk) string { return filepath.Join(s.GetVolumesDir(), disk.GetId()) } @@ -762,7 +775,7 @@ func (s *sPodGuestInstance) getLxcfsMounts() []*runtimeapi.Mount { } } -func (s *sPodGuestInstance) getContainerMounts(input *hostapi.ContainerCreateInput) ([]*runtimeapi.Mount, error) { +func (s *sPodGuestInstance) getContainerMounts(ctrId string, input *hostapi.ContainerCreateInput) ([]*runtimeapi.Mount, error) { inputMounts := input.Spec.VolumeMounts if len(inputMounts) == 0 { return make([]*runtimeapi.Mount, 0), nil @@ -776,7 +789,7 @@ func (s *sPodGuestInstance) getContainerMounts(input *hostapi.ContainerCreateInp SelinuxRelabel: im.SelinuxRelabel, Propagation: volume_mount.GetRuntimeVolumeMountPropagation(im.Propagation), } - hostPath, err := volume_mount.GetDriver(im.Type).GetRuntimeMountHostPath(s, im) + hostPath, err := volume_mount.GetDriver(im.Type).GetRuntimeMountHostPath(s, ctrId, im) if err != nil { return nil, errors.Wrapf(err, "get runtime host mount path of %s", jsonutils.Marshal(im)) } @@ -803,7 +816,7 @@ func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclie if err != nil { return "", errors.Wrap(err, "getPodSandboxConfig") } - mounts, err := s.getContainerMounts(input) + mounts, err := s.getContainerMounts(ctrId, input) if err != nil { return "", errors.Wrap(err, "get container mounts") } diff --git a/pkg/mcclient/options/compute/containers.go b/pkg/mcclient/options/compute/containers.go index c42ddf4731..5ec4013b3b 100644 --- a/pkg/mcclient/options/compute/containers.go +++ b/pkg/mcclient/options/compute/containers.go @@ -199,6 +199,13 @@ func parseContainerVolumeMount(vmStr string) (*apis.ContainerVolumeMount, error) vm.Disk = &apis.ContainerVolumeMountDisk{} } vm.Disk.StorageSizeFile = val + case "overlay": + if vm.Disk == nil { + vm.Disk = &apis.ContainerVolumeMountDisk{} + } + vm.Disk.Overlay = &apis.ContainerVolumeMountDiskOverlay{ + LowerDir: strings.Split(val, ":"), + } } } return vm, nil diff --git a/pkg/mcclient/options/compute/containers_test.go b/pkg/mcclient/options/compute/containers_test.go index 587b63edab..9b9680a65f 100644 --- a/pkg/mcclient/options/compute/containers_test.go +++ b/pkg/mcclient/options/compute/containers_test.go @@ -29,12 +29,17 @@ func Test_parseContainerVolumeMount(t *testing.T) { wantErr bool }{ { - args: "readonly=true,mount_path=/data,disk_index=0", + args: "readonly=true,mount_path=/data,disk_index=0,overlay=/abc:/bca", want: &apis.ContainerVolumeMount{ ReadOnly: true, MountPath: "/data", - Disk: &apis.ContainerVolumeMountDisk{Index: &index0}, - Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK, + Disk: &apis.ContainerVolumeMountDisk{ + Index: &index0, + Overlay: &apis.ContainerVolumeMountDiskOverlay{ + LowerDir: []string{"/abc", "/bca"}, + }, + }, + Type: apis.CONTAINER_VOLUME_MOUNT_TYPE_DISK, }, }, {