From d252c8d682b18eea9610157b59c943c7c7a75fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Mon, 23 Sep 2024 11:49:07 +0800 Subject: [PATCH] fix(region): mount cephfs for pod (#21275) --- pkg/apis/compute/cloudaccount_const.go | 1 + pkg/apis/container.go | 6 ++ pkg/apis/host/container.go | 9 +++ .../container_drivers/volume_mount/cephfs.go | 81 +++++++++++++++++++ pkg/compute/models/cloudaccounts.go | 12 +++ pkg/compute/models/containers.go | 38 +++++++++ pkg/compute/models/filesystem.go | 14 +++- pkg/hostman/container/volume_mount/cephfs.go | 79 ++++++++++++++++++ pkg/mcclient/options/compute/containers.go | 5 ++ 9 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 pkg/compute/container_drivers/volume_mount/cephfs.go create mode 100644 pkg/hostman/container/volume_mount/cephfs.go diff --git a/pkg/apis/compute/cloudaccount_const.go b/pkg/apis/compute/cloudaccount_const.go index cb4e6ca817..17726345ce 100644 --- a/pkg/apis/compute/cloudaccount_const.go +++ b/pkg/apis/compute/cloudaccount_const.go @@ -76,6 +76,7 @@ const ( CLOUD_PROVIDER_GENERICS3 = compute.CLOUD_PROVIDER_GENERICS3 CLOUD_PROVIDER_CEPH = compute.CLOUD_PROVIDER_CEPH CLOUD_PROVIDER_XSKY = compute.CLOUD_PROVIDER_XSKY + CLOUD_PROVIDER_CEPHFS = compute.CLOUD_PROVIDER_CEPHFS CLOUD_PROVIDER_HEALTH_NORMAL = compute.CLOUD_PROVIDER_HEALTH_NORMAL // 远端处于健康状态 CLOUD_PROVIDER_HEALTH_INSUFFICIENT = compute.CLOUD_PROVIDER_HEALTH_INSUFFICIENT // 不足按需资源余额 diff --git a/pkg/apis/container.go b/pkg/apis/container.go index 9c9e5b0f6a..b601e10e9f 100644 --- a/pkg/apis/container.go +++ b/pkg/apis/container.go @@ -121,6 +121,7 @@ const ( CONTAINER_VOLUME_MOUNT_TYPE_DISK ContainerVolumeMountType = "disk" CONTAINER_VOLUME_MOUNT_TYPE_HOST_PATH ContainerVolumeMountType = "host_path" CONTAINER_VOLUME_MOUNT_TYPE_TEXT ContainerVolumeMountType = "text" + CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS ContainerVolumeMountType = "ceph_fs" ) type ContainerDeviceType string @@ -152,6 +153,7 @@ type ContainerVolumeMount struct { Disk *ContainerVolumeMountDisk `json:"disk"` HostPath *ContainerVolumeMountHostPath `json:"host_path"` Text *ContainerVolumeMountText `json:"text"` + CephFS *ContainerVolumeMountCephFS `json:"ceph_fs"` // Mounted read-only if true, read-write otherwise (false or unspecified). ReadOnly bool `json:"read_only"` // Path within the container at which the volume should be mounted. Must @@ -224,3 +226,7 @@ type ContainerVolumeMountHostPath struct { type ContainerVolumeMountText struct { Content string `json:"content"` } + +type ContainerVolumeMountCephFS struct { + Id string `json:"id"` +} diff --git a/pkg/apis/host/container.go b/pkg/apis/host/container.go index 8071dafd39..31ca0ffc24 100644 --- a/pkg/apis/host/container.go +++ b/pkg/apis/host/container.go @@ -27,11 +27,20 @@ type ContainerVolumeMountDisk struct { Overlay *apis.ContainerVolumeMountDiskOverlay `json:"overlay"` } +type ContainerVolumeMountCephFS struct { + Id string `json:"id"` + MonHost string `json:"mon_host"` + Path string `json:"path"` + Secret string `json:"secret"` + Name string `json:"name"` +} + type ContainerVolumeMount struct { Type apis.ContainerVolumeMountType `json:"type"` Disk *ContainerVolumeMountDisk `json:"disk"` HostPath *apis.ContainerVolumeMountHostPath `json:"host_path"` Text *apis.ContainerVolumeMountText `json:"text"` + CephFS *ContainerVolumeMountCephFS `json:"ceph_fs"` // Mounted read-only if true, read-write otherwise (false or unspecified). ReadOnly bool `json:"read_only"` // Path within the container at which the volume should be mounted. Must diff --git a/pkg/compute/container_drivers/volume_mount/cephfs.go b/pkg/compute/container_drivers/volume_mount/cephfs.go new file mode 100644 index 0000000000..5c8c50063f --- /dev/null +++ b/pkg/compute/container_drivers/volume_mount/cephfs.go @@ -0,0 +1,81 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volume_mount + +import ( + "context" + + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func init() { + models.RegisterContainerVolumeMountDriver(newCephFS()) +} + +type cephFS struct{} + +func newCephFS() models.IContainerVolumeMountDriver { + return &cephFS{} +} + +func (h cephFS) GetType() apis.ContainerVolumeMountType { + return apis.CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS +} + +func (h cephFS) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *models.SGuest, vm *apis.ContainerVolumeMount) (*apis.ContainerVolumeMount, error) { + return vm, h.ValidatePodCreateData(ctx, userCred, vm, nil) +} + +func (h cephFS) ValidatePodCreateData(ctx context.Context, userCred mcclient.TokenCredential, vm *apis.ContainerVolumeMount, input *api.ServerCreateInput) error { + fs := vm.CephFS + if fs == nil { + return httperrors.NewNotEmptyError("ceph_fs is nil") + } + if len(fs.Id) == 0 { + return httperrors.NewNotEmptyError("cep_fs.id is empty") + } + obj, err := validators.ValidateModel(ctx, userCred, models.FileSystemManager, &fs.Id) + if err != nil { + return err + } + filesystem := obj.(*models.SFileSystem) + fs.Id = filesystem.Id + if filesystem.Status != apis.STATUS_AVAILABLE { + return httperrors.NewInvalidStatusError("invalid cephfs status %s", filesystem.Status) + } + account := filesystem.GetCloudaccount() + if gotypes.IsNil(account) { + return httperrors.NewInputParameterError("invalid cephfs %s", filesystem.Name) + } + if account.Provider != api.CLOUD_PROVIDER_CEPHFS { + return httperrors.NewInputParameterError("invalid cephfs type %s", account.Provider) + } + if gotypes.IsNil(account.Options) { + return httperrors.NewInputParameterError("missing mon_host") + } + monHost, _ := account.Options.GetString("mon_host") + if len(monHost) == 0 { + return httperrors.NewInputParameterError("missing mon_host") + } + _, err = account.GetOptionPassword() + return err +} diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 96d63a8eea..787f4681a4 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -683,6 +683,18 @@ func (acnt *SCloudaccount) getPassword() (string, error) { return utils.DescryptAESBase64(acnt.Id, acnt.Secret) } +func (acnt *SCloudaccount) GetOptionPassword() (string, error) { + passwd, err := acnt.getPassword() + if err != nil { + return "", err + } + passwdStr, _ := acnt.Options.GetString("password") + if len(passwdStr) == 0 { + return "", fmt.Errorf("missing password") + } + return utils.DescryptAESBase64(passwd, passwdStr) +} + func (acnt *SCloudaccount) regionId() string { if len(acnt.RegionId) > 0 { return acnt.RegionId diff --git a/pkg/compute/models/containers.go b/pkg/compute/models/containers.go index dc2ff6db9e..9e90fdf7ee 100644 --- a/pkg/compute/models/containers.go +++ b/pkg/compute/models/containers.go @@ -16,10 +16,12 @@ package models import ( "context" + "fmt" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/util/sets" "yunion.io/x/sqlchemy" @@ -392,10 +394,39 @@ func (vm *ContainerVolumeMountRelation) toHostDiskMount(disk *apis.ContainerVolu return ret, nil } +func (vm *ContainerVolumeMountRelation) toCephFSMount(fs *apis.ContainerVolumeMountCephFS) (*hostapi.ContainerVolumeMountCephFS, error) { + fsObj, err := FileSystemManager.FetchById(fs.Id) + if err != nil { + return nil, errors.Errorf("fetch cephfs by id %s", fs.Id) + } + + filesystem := fsObj.(*SFileSystem) + ret := &hostapi.ContainerVolumeMountCephFS{ + Id: filesystem.Id, + Path: filesystem.ExternalId, + } + + account := filesystem.GetCloudaccount() + if gotypes.IsNil(account) { + return nil, fmt.Errorf("invalid cephfs %s", filesystem.Name) + } + ret.Secret, err = account.GetOptionPassword() + if err != nil { + return nil, err + } + ret.Name, _ = account.Options.GetString("name") + if len(ret.Name) == 0 { + ret.Name = "admin" + } + ret.MonHost, _ = account.Options.GetString("mon_host") + return ret, nil +} + func (vm *ContainerVolumeMountRelation) ToHostMount(ctx context.Context, userCred mcclient.TokenCredential) (*hostapi.ContainerVolumeMount, error) { ret := &hostapi.ContainerVolumeMount{ Type: vm.VolumeMount.Type, Disk: nil, + CephFS: nil, Text: vm.VolumeMount.Text, HostPath: vm.VolumeMount.HostPath, ReadOnly: vm.VolumeMount.ReadOnly, @@ -412,6 +443,13 @@ func (vm *ContainerVolumeMountRelation) ToHostMount(ctx context.Context, userCre } ret.Disk = disk } + if vm.VolumeMount.CephFS != nil { + fs, err := vm.toCephFSMount(vm.VolumeMount.CephFS) + if err != nil { + return nil, errors.Wrapf(err, "getCephFSSecret") + } + ret.CephFS = fs + } return ret, nil } diff --git a/pkg/compute/models/filesystem.go b/pkg/compute/models/filesystem.go index 8444390d48..da6983ff49 100644 --- a/pkg/compute/models/filesystem.go +++ b/pkg/compute/models/filesystem.go @@ -157,7 +157,19 @@ func (man *SFileSystemManager) ValidateCreateData(ctx context.Context, userCred input.CloudregionId = region.Id if len(input.ManagerId) == 0 { - return input, httperrors.NewMissingParameterError("manager_id") + sq := CloudproviderManager.Query().Equals("provider", api.CLOUD_PROVIDER_CEPHFS).SubQuery() + q := CloudproviderRegionManager.Query().Equals("cloudregion_id", input.CloudregionId) + q = q.Join(sq, sqlchemy.Equals(q.Field("cloudprovider_id"), sq.Field("id"))) + cprgs := []SCloudproviderregion{} + err = q.All(&cprgs) + if err != nil { + return input, err + } + if len(cprgs) == 1 { + input.ManagerId = cprgs[0].CloudproviderId + } else { + return input, httperrors.NewMissingParameterError("manager_id") + } } if len(input.Duration) > 0 { diff --git a/pkg/hostman/container/volume_mount/cephfs.go b/pkg/hostman/container/volume_mount/cephfs.go new file mode 100644 index 0000000000..554bba4cff --- /dev/null +++ b/pkg/hostman/container/volume_mount/cephfs.go @@ -0,0 +1,79 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volume_mount + +import ( + "fmt" + "path/filepath" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + hostapi "yunion.io/x/onecloud/pkg/apis/host" + container_storage "yunion.io/x/onecloud/pkg/hostman/container/storage" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func init() { + RegisterDriver(newCephFS()) +} + +type cephFS struct{} + +func (h cephFS) Mount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + dir, err := h.GetRuntimeMountHostPath(pod, ctrId, vm) + if err != nil { + return err + } + procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", dir).Output() + args := []string{ + "-t", "ceph", + fmt.Sprintf("%s:%s", vm.CephFS.MonHost, vm.CephFS.Path), dir, + "-o", fmt.Sprintf("name=%s,secret=%s", vm.CephFS.Name, vm.CephFS.Secret), + } + out, err := procutils.NewRemoteCommandAsFarAsPossible("mount", args...).Output() + if err != nil { + return errors.Wrapf(err, "mount %s: %s", dir, out) + } + return nil +} + +func (h cephFS) Unmount(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) error { + dir, err := h.GetRuntimeMountHostPath(pod, ctrId, vm) + if err != nil { + return err + } + err = container_storage.Unmount(dir) + if err != nil { + return errors.Wrapf(err, "unmount %s", dir) + } + _, err = procutils.NewRemoteCommandAsFarAsPossible("rm", "-f", dir).Output() + return err +} + +func newCephFS() IVolumeMount { + return &cephFS{} +} + +func (h cephFS) GetType() apis.ContainerVolumeMountType { + return apis.CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS +} + +func (h cephFS) GetRuntimeMountHostPath(pod IPodInfo, ctrId string, vm *hostapi.ContainerVolumeMount) (string, error) { + if vm.CephFS == nil { + return "", fmt.Errorf("cephfs is nil") + } + return filepath.Join(pod.GetVolumesDir(), vm.CephFS.Id), nil +} diff --git a/pkg/mcclient/options/compute/containers.go b/pkg/mcclient/options/compute/containers.go index b514bc8c66..8aeff10dce 100644 --- a/pkg/mcclient/options/compute/containers.go +++ b/pkg/mcclient/options/compute/containers.go @@ -260,6 +260,11 @@ func parseContainerVolumeMount(vmStr string) (*apis.ContainerVolumeMount, error) vm.Text = &apis.ContainerVolumeMountText{ Content: string(content), } + case "cephfs": + vm.Type = apis.CONTAINER_VOLUME_MOUNT_TYPE_CEPHF_FS + vm.CephFS = &apis.ContainerVolumeMountCephFS{ + Id: val, + } } } return vm, nil