diff --git a/cmd/climc/shell/compute/storages.go b/cmd/climc/shell/compute/storages.go index b368fd7f55..878c1462df 100644 --- a/cmd/climc/shell/compute/storages.go +++ b/cmd/climc/shell/compute/storages.go @@ -60,9 +60,10 @@ func init() { info := struct { StorageType string `json:"storage_type"` StorageConf struct { - Key string `json:"key"` - MonHost string `json:"mon_host"` - Pool string `json:"pool"` + Key string `json:"key"` + MonHost string `json:"mon_host"` + Pool string `json:"pool"` + EnableMessengerV2 bool `json:"enable_messenger_v2"` } }{} err = result.Unmarshal(&info) @@ -76,6 +77,7 @@ func init() { info.StorageConf.MonHost, info.StorageConf.Key, info.StorageConf.Pool, + info.StorageConf.EnableMessengerV2, ) if err != nil { return errors.Wrap(err, "cephutils.NewClient") diff --git a/pkg/apis/compute/storage.go b/pkg/apis/compute/storage.go index 6e3ca1a0cc..0b8f2ec5ae 100644 --- a/pkg/apis/compute/storage.go +++ b/pkg/apis/compute/storage.go @@ -77,6 +77,9 @@ type StorageCreateInput struct { // example: 192.168.222.3,192.168.222.4,192.168.222.99 RbdMonHost string `json:"rbd_mon_host"` + // enable ceph messenger v2 + EnableMessengerV2 *bool `json:"enable_messenger_v2"` + // swagger:ignore MonHost string @@ -239,6 +242,9 @@ type StorageUpdateInput struct { // example: AQDigB9dtnDAKhAAxS6X4zi4BPR/lIle4nf4Dw== RbdKey string `json:"rbd_key"` + // enable ceph messenger v2 + EnableMessengerV2 *bool `json:"enable_messenger_v2"` + RbdTimeoutInput // swagger:ignore diff --git a/pkg/compute/storagedrivers/rbd.go b/pkg/compute/storagedrivers/rbd.go index bf6606ed90..5a55de69cf 100644 --- a/pkg/compute/storagedrivers/rbd.go +++ b/pkg/compute/storagedrivers/rbd.go @@ -87,6 +87,10 @@ func (self *SRbdStorageDriver) ValidateCreateData(ctx context.Context, userCred } } + enableMessengerV2 := false + if input.EnableMessengerV2 != nil { + enableMessengerV2 = *input.EnableMessengerV2 + } input.StorageConf.Update( jsonutils.Marshal(map[string]interface{}{ "mon_host": input.MonHost, @@ -95,6 +99,7 @@ func (self *SRbdStorageDriver) ValidateCreateData(ctx context.Context, userCred "rados_mon_op_timeout": input.RadosMonOpTimeout, "rados_osd_op_timeout": input.RadosOsdOpTimeout, "client_mount_timeout": input.ClientMountTimeout, + "enable_messenger_v2": enableMessengerV2, })) return nil } @@ -110,6 +115,11 @@ func (self *SRbdStorageDriver) ValidateUpdateData(ctx context.Context, userCred } } + if input.EnableMessengerV2 != nil { + input.StorageConf.Set("enable_messenger_v2", jsonutils.NewBool(*input.EnableMessengerV2)) + input.UpdateStorageConf = true + } + if len(input.RbdKey) > 0 { input.StorageConf.Set("key", jsonutils.NewString(strings.Trim(input.RbdKey, " "))) input.UpdateStorageConf = true diff --git a/pkg/hostman/storageman/storage_rbd.go b/pkg/hostman/storageman/storage_rbd.go index 4537daf738..1e9d4172aa 100644 --- a/pkg/hostman/storageman/storage_rbd.go +++ b/pkg/hostman/storageman/storage_rbd.go @@ -47,9 +47,13 @@ const ( ) type sStorageConf struct { - MonHost string - Key string - Pool string + MonHost string + Key string + Pool string + // https://docs.ceph.com/en/latest/rados/configuration/msgr2/ + // The messenger v2 protocol, or msgr2, is the second major + // revision on Ceph’s on-wire protocol. + EnableMessengerV2 bool RadosMonOpTimeout int64 RadosOsdOpTimeout int64 ClientMountTimeout int64 @@ -98,7 +102,7 @@ func (s *SRbdStorage) getCephClient(pool string) (*cephutils.CephClient, error) if pool == "" { pool = s.Pool } - return cephutils.NewClient(s.MonHost, s.Key, pool) + return cephutils.NewClient(s.MonHost, s.Key, pool, s.EnableMessengerV2) } func (s *SRbdStorage) getClient() (*cephutils.CephClient, error) { diff --git a/pkg/mcclient/options/compute/storage.go b/pkg/mcclient/options/compute/storage.go index fabc0d8e64..b452668b2e 100644 --- a/pkg/mcclient/options/compute/storage.go +++ b/pkg/mcclient/options/compute/storage.go @@ -54,6 +54,7 @@ type StorageUpdateOptions struct { RbdRadosMonOpTimeout int64 `help:"ceph rados_mon_op_timeout"` RbdRadosOsdOpTimeout int64 `help:"ceph rados_osd_op_timeout"` RbdClientMountTimeout int64 `help:"ceph client_mount_timeout"` + RbdEnableMessengerV2 bool `help:"ceph enable Messenger V2"` RbdKey string `help:"ceph rbd key"` Reserved string `help:"Reserved storage space"` Capacity int `help:"Capacity for storage"` @@ -71,6 +72,7 @@ type StorageCreateOptions struct { MediumType string `help:"Medium type" choices:"ssd|rotate" default:"ssd"` StorageType string `help:"Storage type" choices:"local|nas|vsan|rbd|nfs|gpfs|baremetal|clvm|slvm"` RbdMonHost string `help:"Ceph mon_host config"` + RbdEnableMessengerV2 bool `help:"ceph enable Messenger V2"` RbdRadosMonOpTimeout int64 `help:"ceph rados_mon_op_timeout"` RbdRadosOsdOpTimeout int64 `help:"ceph rados_osd_op_timeout"` RbdClientMountTimeout int64 `help:"ceph client_mount_timeout"` diff --git a/pkg/util/cephutils/ceph.go b/pkg/util/cephutils/ceph.go index ec86029280..0a05a343eb 100644 --- a/pkg/util/cephutils/ceph.go +++ b/pkg/util/cephutils/ceph.go @@ -230,7 +230,7 @@ func (cli *CephClient) SetTimeout(timeout int) { const DEFAULT_TIMTOUT_SECOND = 15 -func NewClient(monHost, key, pool string) (*CephClient, error) { +func NewClient(monHost, key, pool string, enableMessengerV2 bool) (*CephClient, error) { client := &CephClient{ monHost: monHost, key: key, @@ -248,15 +248,24 @@ func NewClient(monHost, key, pool string) (*CephClient, error) { } } monHosts := []string{} - for _, monHost := range strings.Split(client.monHost, ",") { - monHosts = append(monHosts, fmt.Sprintf(`[%s]`, monHost)) + if enableMessengerV2 { + for _, monHost := range strings.Split(client.monHost, ",") { + monHosts = append(monHosts, fmt.Sprintf(`[v2:%s:3300/0,v1:%s:6789/0]`, monHost, monHost)) + } + } else { + monHosts := []string{} + for _, monHost := range strings.Split(client.monHost, ",") { + monHosts = append(monHosts, fmt.Sprintf(`[%s]`, monHost)) + } } + client.monHost = strings.Join(monHosts, ",") + conf := fmt.Sprintf(`[global] mon host = %s rados mon op timeout = 5 rados osd_op timeout = 1200 client mount timeout = 120 -`, strings.Join(monHosts, ",")) +`, client.monHost) if len(client.key) == 0 { conf = fmt.Sprintf(`%s auth_cluster_required = none