fix(region): mount cephfs for pod (#21275)

pkg/apis/compute/cloudaccount_const.go

@@ -76,6 +76,7 @@ const (
 	CLOUD_PROVIDER_GENERICS3 = compute.CLOUD_PROVIDER_GENERICS3
 	CLOUD_PROVIDER_CEPH      = compute.CLOUD_PROVIDER_CEPH
 	CLOUD_PROVIDER_XSKY      = compute.CLOUD_PROVIDER_XSKY
+	CLOUD_PROVIDER_CEPHFS    = compute.CLOUD_PROVIDER_CEPHFS
 
 	CLOUD_PROVIDER_HEALTH_NORMAL        = compute.CLOUD_PROVIDER_HEALTH_NORMAL        // 远端处于健康状态
 	CLOUD_PROVIDER_HEALTH_INSUFFICIENT  = compute.CLOUD_PROVIDER_HEALTH_INSUFFICIENT  // 不足按需资源余额

pkg/apis/container.go

@@ -121,6 +121,7 @@ const (
 	CONTAINER_VOLUME_MOUNT_TYPE_DISK      ContainerVolumeMountType = "disk"
 	CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH ContainerVolumeMountType = "host_path"
 	CONTAINER_VOLUME_MOUNT_TYPE_TEXT      ContainerVolumeMountType = "text"
+	CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS  ContainerVolumeMountType = "ceph_fs"
 )
 
 type ContainerDeviceType string
@@ -152,6 +153,7 @@ type ContainerVolumeMount struct {
 	Disk     *ContainerVolumeMountDisk     `json:"disk"`
 	HostPath *ContainerVolumeMountHostPath `json:"host_path"`
 	Text     *ContainerVolumeMountText     `json:"text"`
+	CephFS   *ContainerVolumeMountCephFS   `json:"ceph_fs"`
 	// Mounted read-only if true, read-write otherwise (false or unspecified).
 	ReadOnly bool `json:"read_only"`
 	// Path within the container at which the volume should be mounted.  Must
@@ -224,3 +226,7 @@ type ContainerVolumeMountHostPath struct {
 type ContainerVolumeMountText struct {
 	Content string `json:"content"`
 }
+
+type ContainerVolumeMountCephFS struct {
+	Id string `json:"id"`
+}

pkg/apis/host/container.go

@@ -27,11 +27,20 @@ type ContainerVolumeMountDisk struct {
 	Overlay         *apis.ContainerVolumeMountDiskOverlay `json:"overlay"`
 }
 
+type ContainerVolumeMountCephFS struct {
+	Id      string `json:"id"`
+	MonHost string `json:"mon_host"`
+	Path    string `json:"path"`
+	Secret  string `json:"secret"`
+	Name    string `json:"name"`
+}
+
 type ContainerVolumeMount struct {
 	Type     apis.ContainerVolumeMountType      `json:"type"`
 	Disk     *ContainerVolumeMountDisk          `json:"disk"`
 	HostPath *apis.ContainerVolumeMountHostPath `json:"host_path"`
 	Text     *apis.ContainerVolumeMountText     `json:"text"`
+	CephFS   *ContainerVolumeMountCephFS        `json:"ceph_fs"`
 	// Mounted read-only if true, read-write otherwise (false or unspecified).
 	ReadOnly bool `json:"read_only"`
 	// Path within the container at which the volume should be mounted.  Must

pkg/compute/container_drivers/volume_mount/cephfs.go

@@ -0,0 +1,81 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package volume_mount
+
+import (
+	"context"
+
+	"yunion.io/x/pkg/gotypes"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func init() {
+	models.RegisterContainerVolumeMountDriver(newCephFS())
+}
+
+type cephFS struct{}
+
+func newCephFS() models.IContainerVolumeMountDriver {
+	return &cephFS{}
+}
+
+func (h cephFS) GetType() apis.ContainerVolumeMountType {
+	return apis.CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS
+}
+
+func (h cephFS) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *models.SGuest, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) {
+	return vm, h.ValidatePodCreateData(ctx, userCred, vm, nil)
+}
+
+func (h cephFS) ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, input *api.ServerCreateInput) error {
+	fs := vm.CephFS
+	if fs == nil {
+		return httperrors.NewNotEmptyError("ceph_fs is nil")
+	}
+	if len(fs.Id) == 0 {
+		return httperrors.NewNotEmptyError("cep_fs.id is empty")
+	}
+	obj, err := validators.ValidateModel(ctx, userCred, models.FileSystemManager, &fs.Id)
+	if err != nil {
+		return err
+	}
+	filesystem := obj.(*models.SFileSystem)
+	fs.Id = filesystem.Id
+	if filesystem.Status != apis.STATUS_AVAILABLE {
+		return httperrors.NewInvalidStatusError("invalid cephfs status %s", filesystem.Status)
+	}
+	account := filesystem.GetCloudaccount()
+	if gotypes.IsNil(account) {
+		return httperrors.NewInputParameterError("invalid cephfs %s", filesystem.Name)
+	}
+	if account.Provider != api.CLOUD_PROVIDER_CEPHFS {
+		return httperrors.NewInputParameterError("invalid cephfs type %s", account.Provider)
+	}
+	if gotypes.IsNil(account.Options) {
+		return httperrors.NewInputParameterError("missing mon_host")
+	}
+	monHost, _ := account.Options.GetString("mon_host")
+	if len(monHost) == 0 {
+		return httperrors.NewInputParameterError("missing mon_host")
+	}
+	_, err = account.GetOptionPassword()
+	return err
+}

pkg/compute/models/cloudaccounts.go

@@ -683,6 +683,18 @@ func (acnt *SCloudaccount) getPassword() (string, error) {
 	return utils.DescryptAESBase64(acnt.Id, acnt.Secret)
 }
 
+func (acnt *SCloudaccount) GetOptionPassword() (string, error) {
+	passwd, err := acnt.getPassword()
+	if err != nil {
+		return "", err
+	}
+	passwdStr, _ := acnt.Options.GetString("password")
+	if len(passwdStr) == 0 {
+		return "", fmt.Errorf("missing password")
+	}
+	return utils.DescryptAESBase64(passwd, passwdStr)
+}
+
 func (acnt *SCloudaccount) regionId() string {
 	if len(acnt.RegionId) > 0 {
 		return acnt.RegionId

pkg/compute/models/containers.go

@@ -16,10 +16,12 @@ package models
 
 import (
 	"context"
+	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/sqlchemy"
 
@@ -392,10 +394,39 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu
 	return ret, nil
 }
 
+func (vm *ContainerVolumeMountRelation) toCephFSMount(fs *apis.ContainerVolumeMountCephFS) (*hostapi.ContainerVolumeMountCephFS, error) {
+	fsObj, err := FileSystemManager.FetchById(fs.Id)
+	if err != nil {
+		return nil, errors.Errorf("fetch cephfs by id %s", fs.Id)
+	}
+
+	filesystem := fsObj.(*SFileSystem)
+	ret := &hostapi.ContainerVolumeMountCephFS{
+		Id:   filesystem.Id,
+		Path: filesystem.ExternalId,
+	}
+
+	account := filesystem.GetCloudaccount()
+	if gotypes.IsNil(account) {
+		return nil, fmt.Errorf("invalid cephfs %s", filesystem.Name)
+	}
+	ret.Secret, err = account.GetOptionPassword()
+	if err != nil {
+		return nil, err
+	}
+	ret.Name, _ = account.Options.GetString("name")
+	if len(ret.Name) == 0 {
+		ret.Name = "admin"
+	}
+	ret.MonHost, _ = account.Options.GetString("mon_host")
+	return ret, nil
+}
+
 func (vm *ContainerVolumeMountRelation) ToHostMount(ctx context.Context, userCred mcclient.TokenCredential) (*hostapi.ContainerVolumeMount, error) {
 	ret := &hostapi.ContainerVolumeMount{
 		Type:           vm.VolumeMount.Type,
 		Disk:           nil,
+		CephFS:         nil,
 		Text:           vm.VolumeMount.Text,
 		HostPath:       vm.VolumeMount.HostPath,
 		ReadOnly:       vm.VolumeMount.ReadOnly,
@@ -412,6 +443,13 @@ func (vm *ContainerVolumeMountRelation) ToHostMount(ctx context.Context, userCre
 		}
 		ret.Disk = disk
 	}
+	if vm.VolumeMount.CephFS != nil {
+		fs, err := vm.toCephFSMount(vm.VolumeMount.CephFS)
+		if err != nil {
+			return nil, errors.Wrapf(err, "getCephFSSecret")
+		}
+		ret.CephFS = fs
+	}
 	return ret, nil
 }
 

pkg/compute/models/filesystem.go

@@ -157,7 +157,19 @@ func (man *SFileSystemManager) ValidateCreateData(ctx context.Context, userCred
 	input.CloudregionId = region.Id
 
 	if len(input.ManagerId) == 0 {
-		return input, httperrors.NewMissingParameterError("manager_id")
+		sq := CloudproviderManager.Query().Equals("provider", api.CLOUD_PROVIDER_CEPHFS).SubQuery()
+		q := CloudproviderRegionManager.Query().Equals("cloudregion_id", input.CloudregionId)
+		q = q.Join(sq, sqlchemy.Equals(q.Field("cloudprovider_id"), sq.Field("id")))
+		cprgs := []SCloudproviderregion{}
+		err = q.All(&cprgs)
+		if err != nil {
+			return input, err
+		}
+		if len(cprgs) == 1 {
+			input.ManagerId = cprgs[0].CloudproviderId
+		} else {
+			return input, httperrors.NewMissingParameterError("manager_id")
+		}
 	}
 
 	if len(input.Duration) > 0 {

pkg/hostman/container/volume_mount/cephfs.go

@@ -0,0 +1,79 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package volume_mount
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+func init() {
+	RegisterDriver(newCephFS())
+}
+
+type cephFS struct{}
+
+func (h cephFS) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error {
+	dir, err := h.GetRuntimeMountHostPath(pod, ctrId, vm)
+	if err != nil {
+		return err
+	}
+	procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output()
+	args := []string{
+		"-t", "ceph",
+		fmt.Sprintf("%s:%s", vm.CephFS.MonHost, vm.CephFS.Path), dir,
+		"-o", fmt.Sprintf("name=%s,secret=%s", vm.CephFS.Name, vm.CephFS.Secret),
+	}
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", args...).Output()
+	if err != nil {
+		return errors.Wrapf(err, "mount %s: %s", dir, out)
+	}
+	return nil
+}
+
+func (h cephFS) Unmount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error {
+	dir, err := h.GetRuntimeMountHostPath(pod, ctrId, vm)
+	if err != nil {
+		return err
+	}
+	err = container_storage.Unmount(dir)
+	if err != nil {
+		return errors.Wrapf(err, "unmount %s", dir)
+	}
+	_, err = procutils.NewRemoteCommandAsFarAsPossible("rm", "-f", dir).Output()
+	return err
+}
+
+func newCephFS() IVolumeMount {
+	return &cephFS{}
+}
+
+func (h cephFS) GetType() apis.ContainerVolumeMountType {
+	return apis.CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS
+}
+
+func (h cephFS) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) (string, error) {
+	if vm.CephFS == nil {
+		return "", fmt.Errorf("cephfs is nil")
+	}
+	return filepath.Join(pod.GetVolumesDir(), vm.CephFS.Id), nil
+}

pkg/mcclient/options/compute/containers.go

@@ -260,6 +260,11 @@ func parseContainerVolumeMount(vmStr string) (*apis.ContainerVolumeMount, error)
 			vm.Text = &apis.ContainerVolumeMountText{
 				Content: string(content),
 			}
+		case "cephfs":
+			vm.Type = apis.CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS
+			vm.CephFS = &apis.ContainerVolumeMountCephFS{
+				Id: val,
+			}
 		}
 	}
 	return vm, nil