Merge pull request #19769 from zexi/container-lifecycle-devices

Container lifecycle devices

pkg/apis/compute/api.go

@@ -287,6 +287,7 @@ type IsolatedDeviceConfig struct {
 	NetworkIndex *int8  `json:"network_index"`
 	WireId       string `json:"wire_id"`
 	DiskIndex    *int8  `json:"disk_index"`
+	DevicePath   string `json:"device_path"`
 }
 
 type BaremetalDiskConfig struct {

pkg/apis/container.go

@@ -21,6 +21,25 @@ type ContainerKeyValue struct {
 	Value string `json:"value"`
 }
 
+type ContainerLifecyleHandlerType string
+
+const (
+	ContainerLifecyleHandlerTypeExec ContainerLifecyleHandlerType = "exec"
+)
+
+type ContainerLifecyleHandlerExecAction struct {
+	Command []string `json:"command"`
+}
+
+type ContainerLifecyleHandler struct {
+	Type ContainerLifecyleHandlerType        `json:"type"`
+	Exec *ContainerLifecyleHandlerExecAction `json:"exec"`
+}
+
+type ContainerLifecyle struct {
+	PostStart *ContainerLifecyleHandler `json:"post_start"`
+}
+
 type ContainerSpec struct {
 	// Image to use.
 	Image string `json:"image"`
@@ -37,9 +56,11 @@ type ContainerSpec struct {
 	// Enable lxcfs
 	EnableLxcfs bool `json:"enable_lxcfs"`
 	// Volume mounts
-	VolumeMounts []*ContainerVolumeMount `json:"volume_mounts"`
-	Capabilities *ContainerCapability    `json:"capabilities"`
-	Privileged   bool                    `json:"privileged"`
+	VolumeMounts       []*ContainerVolumeMount `json:"volume_mounts"`
+	Capabilities       *ContainerCapability    `json:"capabilities"`
+	Privileged         bool                    `json:"privileged"`
+	Lifecyle           *ContainerLifecyle      `json:"lifecyle"`
+	CgroupDevicesAllow []string                `json:"cgroup_devices_allow"`
 }
 
 type ContainerCapability struct {

pkg/cloudcommon/cmdline/parser.go

@@ -362,6 +362,8 @@ func ParseIsolatedDevice(desc string, idx int) (*compute.IsolatedDeviceConfig, e
 			dev.DevType = p
 		} else if strings.HasPrefix(p, "vendor=") {
 			dev.Vendor = p[len("vendor="):]
+		} else if strings.HasPrefix(p, "device_path=") {
+			dev.DevicePath = p[len("device_path="):]
 		} else {
 			dev.Model = p
 		}

pkg/cloudcommon/cmdline/parser_test.go

@@ -195,3 +195,49 @@ func TestFetchDiskConfigsByJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestParseIsolatedDevice(t *testing.T) {
+	tests := []struct {
+		desc    string
+		idx     int
+		want    *compute.IsolatedDeviceConfig
+		wantErr bool
+	}{
+		{
+			desc: "device_path=/dev/nvme0",
+			idx:  0,
+			want: &compute.IsolatedDeviceConfig{
+				DevicePath: "/dev/nvme0",
+			},
+		},
+		{
+			desc: "GPU_HPC:device_path=/dev/nvme0",
+			idx:  0,
+			want: &compute.IsolatedDeviceConfig{
+				DevicePath: "/dev/nvme0",
+				Model:      "GPU_HPC",
+			},
+		},
+		{
+			desc: "GPU_HPC:device_path=/dev/nvme0:1d3ce781-2b64-4ee7-8d23-6bbcf478c54b",
+			idx:  0,
+			want: &compute.IsolatedDeviceConfig{
+				Id:         "1d3ce781-2b64-4ee7-8d23-6bbcf478c54b",
+				DevicePath: "/dev/nvme0",
+				Model:      "GPU_HPC",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			got, err := ParseIsolatedDevice(tt.desc, tt.idx)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseIsolatedDevice() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ParseIsolatedDevice() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/compute/container_drivers/lifecycle/doc.go

@@ -0,0 +1 @@
+package lifecycle // import "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle"

pkg/compute/container_drivers/lifecycle/exec.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lifecycle
+
+import (
+	"context"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func init() {
+	models.RegisterContainerLifecyleDriver(newExec())
+}
+
+type execDriver struct{}
+
+func newExec() models.IContainerLifecyleDriver {
+	return &execDriver{}
+}
+
+func (e execDriver) GetType() apis.ContainerLifecyleHandlerType {
+	return apis.ContainerLifecyleHandlerTypeExec
+}
+
+func (e execDriver) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error {
+	if input.Exec == nil {
+		return httperrors.NewNotEmptyError("exec field")
+	}
+	if len(input.Exec.Command) == 0 {
+		return httperrors.NewNotEmptyError("command is required")
+	}
+	return nil
+}

pkg/compute/models/container_drivers.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"context"
+	"sync"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -11,28 +12,67 @@ import (
 )
 
 var (
-	containerVolumeDrivers = make(map[apis.ContainerVolumeMountType]IContainerVolumeMountDriver)
-	containerDeviceDrivers = make(map[apis.ContainerDeviceType]IContainerDeviceDriver)
+	containerVolumeDrivers    = newContainerDrivers()
+	containerDeviceDrivers    = newContainerDrivers()
+	containerLifecycleDrivers = newContainerDrivers()
 )
 
-func RegisterContainerVolumeMountDriver(drv IContainerVolumeMountDriver) {
-	containerVolumeDrivers[drv.GetType()] = drv
+type containerDrivers struct {
+	drivers *sync.Map
 }
 
-func GetContainerVolumeMountDriver(typ apis.ContainerVolumeMountType) IContainerVolumeMountDriver {
-	drv, err := GetContainerVolumeMountDriverWithError(typ)
+func newContainerDrivers() *containerDrivers {
+	return &containerDrivers{
+		drivers: new(sync.Map),
+	}
+}
+
+func (cd *containerDrivers) GetWithError(typ string) (interface{}, error) {
+	drv, ok := cd.drivers.Load(typ)
+	if !ok {
+		return drv, httperrors.NewNotFoundError("not found driver by type %q", typ)
+	}
+	return drv, nil
+}
+
+func (cd *containerDrivers) Get(typ string) interface{} {
+	drv, err := cd.GetWithError(typ)
 	if err != nil {
 		panic(err.Error())
 	}
 	return drv
 }
 
-func GetContainerVolumeMountDriverWithError(typ apis.ContainerVolumeMountType) (IContainerVolumeMountDriver, error) {
-	drv, ok := containerVolumeDrivers[typ]
-	if !ok {
-		return nil, httperrors.NewNotFoundError("not found driver by type %q", typ)
+func (cd *containerDrivers) Register(typ string, drv interface{}) {
+	cd.drivers.Store(typ, drv)
+}
+
+func registerContainerDriver[K ~string, D any](drvs *containerDrivers, typ K, drv D) {
+	drvs.Register(string(typ), drv)
+}
+
+func getContainerDriver[K ~string, D any](drvs *containerDrivers, typ K) D {
+	return drvs.Get(string(typ)).(D)
+}
+
+func getContainerDriverWithError[K ~string, D any](drvs *containerDrivers, typ K) (D, error) {
+	drv, err := drvs.GetWithError(string(typ))
+	if err != nil {
+		return drv.(D), err
 	}
-	return drv, nil
+	return drv.(D), nil
+}
+
+func RegisterContainerVolumeMountDriver(drv IContainerVolumeMountDriver) {
+	registerContainerDriver(containerVolumeDrivers, drv.GetType(), drv)
+}
+
+func GetContainerVolumeMountDriver(typ apis.ContainerVolumeMountType) IContainerVolumeMountDriver {
+	return getContainerDriver[apis.ContainerVolumeMountType, IContainerVolumeMountDriver](containerVolumeDrivers, typ)
+}
+
+func GetContainerVolumeMountDriverWithError(typ apis.ContainerVolumeMountType) (IContainerVolumeMountDriver, error) {
+	return getContainerDriverWithError[apis.ContainerVolumeMountType, IContainerVolumeMountDriver](containerVolumeDrivers, typ)
 }
 
 type IContainerVolumeMountDriver interface {
@@ -42,23 +82,15 @@ type IContainerVolumeMountDriver interface {
 }
 
 func RegisterContainerDeviceDriver(drv IContainerDeviceDriver) {
-	containerDeviceDrivers[drv.GetType()] = drv
+	registerContainerDriver(containerDeviceDrivers, drv.GetType(), drv)
 }
 
 func GetContainerDeviceDriver(typ apis.ContainerDeviceType) IContainerDeviceDriver {
-	drv, err := GetContainerDeviceDriverWithError(typ)
-	if err != nil {
-		panic(err.Error())
-	}
-	return drv
+	return getContainerDriver[apis.ContainerDeviceType, IContainerDeviceDriver](containerDeviceDrivers, typ)
 }
 
 func GetContainerDeviceDriverWithError(typ apis.ContainerDeviceType) (IContainerDeviceDriver, error) {
-	drv, ok := containerDeviceDrivers[typ]
-	if !ok {
-		return nil, httperrors.NewNotFoundError("not found driver by type %q", typ)
-	}
-	return drv, nil
+	return getContainerDriverWithError[apis.ContainerDeviceType, IContainerDeviceDriver](containerDeviceDrivers, typ)
 }
 
 type IContainerDeviceDriver interface {
@@ -67,3 +99,20 @@ type IContainerDeviceDriver interface {
 	ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error)
 	ToHostDevice(dev *api.ContainerDevice) (*hostapi.ContainerDevice, error)
 }
+
+type IContainerLifecyleDriver interface {
+	GetType() apis.ContainerLifecyleHandlerType
+	ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error
+}
+
+func RegisterContainerLifecyleDriver(drv IContainerLifecyleDriver) {
+	registerContainerDriver(containerLifecycleDrivers, drv.GetType(), drv)
+}
+
+func GetContainerLifecyleDriver(typ apis.ContainerLifecyleHandlerType) IContainerLifecyleDriver {
+	return getContainerDriver[apis.ContainerLifecyleHandlerType, IContainerLifecyleDriver](containerLifecycleDrivers, typ)
+}
+
+func GetContainerLifecyleDriverWithError(typ apis.ContainerLifecyleHandlerType) (IContainerLifecyleDriver, error) {
+	return getContainerDriverWithError[apis.ContainerLifecyleHandlerType, IContainerLifecyleDriver](containerLifecycleDrivers, typ)
+}

pkg/compute/models/containers.go

@@ -148,13 +148,35 @@ func (m *SContainerManager) ValidateSpec(ctx context.Context, userCred mcclient.
 		}
 	}
 
+	if err := m.ValidateSpecLifecycle(ctx, userCred, spec); err != nil {
+		return errors.Wrap(err, "validate lifecycle")
+	}
+
 	return nil
 }
 
+func (m *SContainerManager) ValidateSpecLifecycle(ctx context.Context, cred mcclient.TokenCredential, spec *api.ContainerSpec) error {
+	if spec.Lifecyle == nil {
+		return nil
+	}
+	if err := m.ValidateSpecLifecyclePostStart(ctx, cred, spec.Lifecyle.PostStart); err != nil {
+		return errors.Wrap(err, "validate post start")
+	}
+	return nil
+}
+
+func (m *SContainerManager) ValidateSpecLifecyclePostStart(ctx context.Context, userCred mcclient.TokenCredential, input *apis.ContainerLifecyleHandler) error {
+	drv, err := GetContainerLifecyleDriverWithError(input.Type)
+	if err != nil {
+		return httperrors.NewInputParameterError("get lifecycle driver: %v", err)
+	}
+	return drv.ValidateCreateData(ctx, userCred, input)
+}
+
 func (m *SContainerManager) ValidateSpecDevice(ctx context.Context, userCred mcclient.TokenCredential, pod *SGuest, dev *api.ContainerDevice) (*api.ContainerDevice, error) {
 	drv, err := GetContainerDeviceDriverWithError(dev.Type)
 	if err != nil {
-		return nil, httperrors.NewInputParameterError("get device drvice: %v", err)
+		return nil, httperrors.NewInputParameterError("get device driver: %v", err)
 	}
 	return drv.ValidateCreateData(ctx, userCred, pod, dev)
 }

pkg/compute/models/isolated_devices.go

@@ -556,6 +556,8 @@ func (manager *SIsolatedDeviceManager) _isValidDeviceInfo(config *api.IsolatedDe
 func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDesc(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
 	if len(devConfig.Id) > 0 {
 		return manager.attachSpecificDeviceToGuest(ctx, guest, devConfig, userCred)
+	} else if len(devConfig.DevicePath) > 0 {
+		return manager.attachHostDeviceToGuestByDevicePath(ctx, guest, host, devConfig, userCred)
 	} else {
 		return manager.attachHostDeviceToGuestByModel(ctx, guest, host, devConfig, userCred)
 	}
@@ -573,6 +575,19 @@ func (manager *SIsolatedDeviceManager) attachSpecificDeviceToGuest(ctx context.C
 	return guest.attachIsolatedDevice(ctx, userCred, dev, devConfig.NetworkIndex, devConfig.DiskIndex)
 }
 
+func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
+	if len(devConfig.Model) == 0 || len(devConfig.DevicePath) == 0 {
+		return fmt.Errorf("Model or DevicePath is empty: %#v", devConfig)
+	}
+	// if dev type is not nic, wire is empty string
+	devs, err := manager.findHostUnusedByDevAttr(devConfig.Model, "device_path", devConfig.DevicePath, host.Id, devConfig.WireId)
+	if err != nil || len(devs) == 0 {
+		return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id)
+	}
+	selectedDev := devs[0]
+	return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex)
+}
+
 func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel(ctx context.Context, guest *SGuest, host *SHost, devConfig *api.IsolatedDeviceConfig, userCred mcclient.TokenCredential) error {
 	if len(devConfig.Model) == 0 {
 		return fmt.Errorf("Not found model from info: %#v", devConfig)
@@ -642,11 +657,15 @@ func (manager *SIsolatedDeviceManager) FindUnusedGpusOnHost(hostId string) ([]SI
 }
 
 func (manager *SIsolatedDeviceManager) findHostUnusedByDevConfig(model, devType, hostId, wireId string) ([]SIsolatedDevice, error) {
+	return manager.findHostUnusedByDevAttr(model, "dev_type", devType, hostId, wireId)
+}
+
+func (manager *SIsolatedDeviceManager) findHostUnusedByDevAttr(model, attrKey, attrVal, hostId, wireId string) ([]SIsolatedDevice, error) {
 	devs := make([]SIsolatedDevice, 0)
 	q := manager.findUnusedQuery()
 	q = q.Equals("model", model).Equals("host_id", hostId)
-	if devType != "" {
-		q.Equals("dev_type", devType)
+	if attrVal != "" {
+		q.Equals(attrKey, attrVal)
 	}
 	if wireId != "" {
 		wire := WireManager.FetchWireById(wireId)

pkg/compute/service/service.go

@@ -40,6 +40,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	_ "yunion.io/x/onecloud/pkg/compute/container_drivers/device"
+	_ "yunion.io/x/onecloud/pkg/compute/container_drivers/lifecycle"
 	_ "yunion.io/x/onecloud/pkg/compute/container_drivers/volume_mount"
 	_ "yunion.io/x/onecloud/pkg/compute/guestdrivers"
 	_ "yunion.io/x/onecloud/pkg/compute/hostdrivers"

pkg/hostman/container/lifecycle/doc.go

@@ -0,0 +1 @@
+package lifecycle // import "yunion.io/x/onecloud/pkg/hostman/container/lifecycle"

pkg/hostman/container/lifecycle/exec.go

@@ -0,0 +1,44 @@
+package lifecycle
+
+import (
+	"context"
+
+	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/util/pod"
+)
+
+func init() {
+	RegisterDriver(newExec())
+}
+
+type execDriver struct{}
+
+func newExec() ILifecycle {
+	return &execDriver{}
+}
+
+func (e execDriver) GetType() apis.ContainerLifecyleHandlerType {
+	return apis.ContainerLifecyleHandlerTypeExec
+}
+
+func (e execDriver) Run(ctx context.Context, input *apis.ContainerLifecyleHandler, cri pod.CRI, id string) error {
+	cfg := input.Exec
+	cli := cri.GetRuntimeClient()
+	resp, err := cli.ExecSync(ctx, &runtimeapi.ExecSyncRequest{
+		ContainerId: id,
+		Cmd:         cfg.Command,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "exec sync")
+	}
+	if resp.GetExitCode() != 0 {
+		return errors.Wrapf(err, "stdout: %s, stderr: %s, exited: %d", resp.GetStdout(), resp.GetStderr(), resp.GetExitCode())
+	}
+	log.Infof("run command %v: stdout: %s, stderr: %s", cfg.Command, resp.Stdout, resp.Stderr)
+	return nil
+}

pkg/hostman/container/lifecycle/lifecycle.go

@@ -0,0 +1,30 @@
+package lifecycle
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/util/pod"
+)
+
+var (
+	drivers = make(map[apis.ContainerLifecyleHandlerType]ILifecycle)
+)
+
+func RegisterDriver(drv ILifecycle) {
+	drivers[drv.GetType()] = drv
+}
+
+func GetDriver(typ apis.ContainerLifecyleHandlerType) ILifecycle {
+	drv, ok := drivers[typ]
+	if !ok {
+		panic(fmt.Sprintf("not found driver by type %s", typ))
+	}
+	return drv
+}
+
+type ILifecycle interface {
+	GetType() apis.ContainerLifecyleHandlerType
+	Run(ctx context.Context, input *apis.ContainerLifecyleHandler, cri pod.CRI, id string) error
+}

pkg/hostman/guestman/pod.go

@@ -33,6 +33,7 @@ import (
 	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
 	"yunion.io/x/onecloud/pkg/hostman/container/device"
+	"yunion.io/x/onecloud/pkg/hostman/container/lifecycle"
 	"yunion.io/x/onecloud/pkg/hostman/container/volume_mount"
 	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
@@ -47,6 +48,7 @@ import (
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/netutils2/getport"
 	"yunion.io/x/onecloud/pkg/util/pod"
+	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 type PodInstance interface {
@@ -444,9 +446,15 @@ func (s *sPodGuestInstance) stopPod(ctx context.Context, timeout int64) error {
 
 	ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
 	defer cancel()
-	return s.getCRI().StopPod(ctx, &runtimeapi.StopPodSandboxRequest{
+	/*if err := s.getCRI().StopPod(ctx, &runtimeapi.StopPodSandboxRequest{
 		PodSandboxId: s.getCRIId(),
-	})
+	}); err != nil {
+		return errors.Wrapf(err, "stop cri pod: %s", s.getCRIId())
+	}*/
+	if err := s.getCRI().RemovePod(ctx, s.getCRIId()); err != nil {
+		return errors.Wrapf(err, "remove cri pod: %s", s.getCRIId())
+	}
+	return nil
 }
 
 func (s *sPodGuestInstance) LoadDesc() error {
@@ -528,9 +536,30 @@ func (s *sPodGuestInstance) StartContainer(ctx context.Context, userCred mcclien
 	if err := s.getCRI().StartContainer(ctx, criId); err != nil {
 		return nil, errors.Wrap(err, "CRI.StartContainer")
 	}
+	if err := s.setContainerCgroupDevicesAllow(criId, input.Spec.CgroupDevicesAllow); err != nil {
+		return nil, errors.Wrap(err, "set cgroup devices allow")
+	}
+	if err := s.doContainerStartPostLifecycle(ctx, criId, input); err != nil {
+		return nil, errors.Wrap(err, "do container lifecycle")
+	}
 	return nil, nil
 }
 
+func (s *sPodGuestInstance) doContainerStartPostLifecycle(ctx context.Context, criId string, input *hostapi.ContainerCreateInput) error {
+	ls := input.Spec.Lifecyle
+	if ls == nil {
+		return nil
+	}
+	if ls.PostStart == nil {
+		return nil
+	}
+	drv := lifecycle.GetDriver(ls.PostStart.Type)
+	if err := drv.Run(ctx, ls.PostStart, s.getCRI(), criId); err != nil {
+		return errors.Wrapf(err, "run %s", ls.PostStart.Type)
+	}
+	return nil
+}
+
 func (s *sPodGuestInstance) StopContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	criId, err := s.getContainerCRIId(ctrId)
 	if err != nil {
@@ -736,6 +765,26 @@ func (s *sPodGuestInstance) getContainerMounts(input *hostapi.ContainerCreateInp
 	return mounts, nil
 }
 
+func (s *sPodGuestInstance) getContainerCgroupDir(dirType string, ctrId string) string {
+	cgroupDir := "/sys/fs/cgroup"
+	return filepath.Join(cgroupDir, dirType, s.getCgroupParent(), ctrId)
+}
+
+func (s *sPodGuestInstance) getContainerCgroupDevicesDir(ctrId string) string {
+	return s.getContainerCgroupDir("devices", ctrId)
+}
+
+func (s *sPodGuestInstance) setContainerCgroupDevicesAllow(ctrId string, allowStrs []string) error {
+	for _, allowStr := range allowStrs {
+		deviceAllowFile := filepath.Join(s.getContainerCgroupDevicesDir(ctrId), "devices.allow")
+		out, err := procutils.NewRemoteCommandAsFarAsPossible("sh", "-c", fmt.Sprintf("echo '%s' > %s", allowStr, deviceAllowFile)).Output()
+		if err != nil {
+			return errors.Wrapf(err, "echo %s to %s: %s", deviceAllowFile, allowStr, out)
+		}
+	}
+	return nil
+}
+
 func (s *sPodGuestInstance) createContainer(ctx context.Context, userCred mcclient.TokenCredential, ctrId string, input *hostapi.ContainerCreateInput) (string, error) {
 	log.Infof("=====container input: %s", jsonutils.Marshal(input).PrettyString())
 	podCfg, err := s.getPodSandboxConfig()

pkg/mcclient/options/compute/containers.go

@@ -44,31 +44,44 @@ type ContainerDeleteOptions struct {
 }
 
 type ContainerCreateCommonOptions struct {
-	IMAGE       string   `help:"Image of container" json:"image"`
-	Command     []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"`
-	Args        []string `help:"Args for the Command (i.e. command for docker)" json:"args"`
-	WorkingDir  string   `help:"Current working directory of the command" json:"working_dir"`
-	Env         []string `help:"List of environment variable to set in the container and the format is: <key>=<value>"`
-	VolumeMount []string `help:"Volume mount of the container and the format is: name=<val>,mount=<container_path>,readonly=<true_or_false>,disk_index=<disk_number>,disk_id=<disk_id>"`
-	Device      []string `help:"Host device: <host_path>:<container_path>:<permissions>, e.g.: /dev/snd:/dev/snd:rwm"`
-	Privileged  bool     `help:"Privileged mode"`
-	Caps        string   `help:"Container capabilities, e.g.: SETPCAP,AUDIT_WRITE,SYS_CHROOT,CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID,SYSLOG,SYS_ADMIN,WAKE_ALARM,SYS_PTRACE,BLOCK_SUSPEND,MKNOD,KILL,SYS_RESOURCE,NET_RAW,NET_ADMIN,NET_BIND_SERVICE,SYS_NICE"`
-	DropCaps    string   `help:"Container dropped capabilities, split by ','"`
-	EnableLxcfs bool     `help:"Enable lxcfs"`
+	IMAGE             string   `help:"Image of container" json:"image"`
+	Command           []string `help:"Command to execute (i.e., entrypoint for docker)" json:"command"`
+	Args              []string `help:"Args for the Command (i.e. command for docker)" json:"args"`
+	WorkingDir        string   `help:"Current working directory of the command" json:"working_dir"`
+	Env               []string `help:"List of environment variable to set in the container and the format is: <key>=<value>"`
+	VolumeMount       []string `help:"Volume mount of the container and the format is: name=<val>,mount=<container_path>,readonly=<true_or_false>,disk_index=<disk_number>,disk_id=<disk_id>"`
+	Device            []string `help:"Host device: <host_path>:<container_path>:<permissions>, e.g.: /dev/snd:/dev/snd:rwm"`
+	Privileged        bool     `help:"Privileged mode"`
+	Caps              string   `help:"Container capabilities, e.g.: SETPCAP,AUDIT_WRITE,SYS_CHROOT,CHOWN,DAC_OVERRIDE,FOWNER,SETGID,SETUID,SYSLOG,SYS_ADMIN,WAKE_ALARM,SYS_PTRACE,BLOCK_SUSPEND,MKNOD,KILL,SYS_RESOURCE,NET_RAW,NET_ADMIN,NET_BIND_SERVICE,SYS_NICE"`
+	DropCaps          string   `help:"Container dropped capabilities, split by ','"`
+	EnableLxcfs       bool     `help:"Enable lxcfs"`
+	PostStartExec     string   `help:"Post started execution command"`
+	CgroupDeviceAllow []string `help:"Cgroup devices.allow, e.g.: 'c 13:* rwm'"`
 }
 
 func (o ContainerCreateCommonOptions) getCreateSpec() (*computeapi.ContainerSpec, error) {
 	req := &computeapi.ContainerSpec{
 		ContainerSpec: apis.ContainerSpec{
-			Image:        o.IMAGE,
-			Command:      o.Command,
-			Args:         o.Args,
-			WorkingDir:   o.WorkingDir,
-			EnableLxcfs:  o.EnableLxcfs,
-			Privileged:   o.Privileged,
-			Capabilities: &apis.ContainerCapability{},
+			Image:              o.IMAGE,
+			Command:            o.Command,
+			Args:               o.Args,
+			WorkingDir:         o.WorkingDir,
+			EnableLxcfs:        o.EnableLxcfs,
+			Privileged:         o.Privileged,
+			Capabilities:       &apis.ContainerCapability{},
+			CgroupDevicesAllow: o.CgroupDeviceAllow,
 		},
 	}
+	if len(o.PostStartExec) != 0 {
+		req.Lifecyle = &apis.ContainerLifecyle{
+			PostStart: &apis.ContainerLifecyleHandler{
+				Type: apis.ContainerLifecyleHandlerTypeExec,
+				Exec: &apis.ContainerLifecyleHandlerExecAction{
+					Command: strings.Split(o.PostStartExec, " "),
+				},
+			},
+		}
+	}
 	if len(o.Caps) != 0 {
 		req.Capabilities.Add = strings.Split(o.Caps, ",")
 	}

pkg/scheduler/algorithm/predicates/isolated_device_predicate.go

@@ -146,6 +146,25 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c
 		}
 	}
 
+	// check host device by device_path
+	devicePathReq := make(map[string]int, 0)
+	for _, dev := range reqIsoDevs {
+		if len(dev.DevicePath) != 0 {
+			devicePathReq[dev.DevicePath] += 1
+		}
+	}
+	for devPath, reqCnt := range devicePathReq {
+		freeCount := len(getter.UnusedIsolatedDevicesByDevicePath(devPath))
+		if freeCount < reqCount {
+			h.Exclude(fmt.Sprintf("IsolatedDevice device_path %q not enough, request: %d, hostFree: %d", devPath, reqCount, freeCount))
+			return h.GetResult()
+		}
+		cap := freeCount / reqCnt
+		if int64(cap) < minCapacity {
+			minCapacity = int64(cap)
+		}
+	}
+
 	h.SetCapacity(minCapacity)
 	return h.GetResult()
 }

pkg/scheduler/cache/candidate/base.go

@@ -299,6 +299,10 @@ func (b baseHostGetter) UnusedIsolatedDevicesByVendorModel(vendorModel string) [
 	return b.h.UnusedIsolatedDevicesByVendorModel(vendorModel)
 }
 
+func (b baseHostGetter) UnusedIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc {
+	return b.h.UnusedIsolatedDevicesByDevicePath(devPath)
+}
+
 func (b baseHostGetter) UnusedIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc {
 	return b.h.UnusedIsolatedDevicesByModel(model)
 }
@@ -507,6 +511,16 @@ func (h *BaseHostDesc) UnusedIsolatedDevicesByModel(model string) []*core.Isolat
 	return ret
 }
 
+func (h *BaseHostDesc) UnusedIsolatedDevicesByDevicePath(devPath string) []*core.IsolatedDeviceDesc {
+	ret := make([]*core.IsolatedDeviceDesc, 0)
+	for _, dev := range h.UnusedIsolatedDevices() {
+		if devPath == dev.DevicePath {
+			ret = append(ret, dev)
+		}
+	}
+	return ret
+}
+
 func (h *BaseHostDesc) UnusedIsolatedDevicesByModelAndWire(model, wire string) []*core.IsolatedDeviceDesc {
 	ret := make([]*core.IsolatedDeviceDesc, 0)
 	for _, dev := range h.UnusedIsolatedDevices() {
@@ -558,6 +572,7 @@ func (h *BaseHostDesc) fillIsolatedDevices(b *baseBuilder, host *computemodels.S
 			Addr:           devModel.Addr,
 			VendorDeviceID: devModel.VendorDeviceId,
 			WireId:         devModel.WireId,
+			DevicePath:     devModel.DevicePath,
 		}
 		devs[index] = dev
 	}

pkg/scheduler/core/types.go

@@ -122,6 +122,7 @@ type CandidatePropertyGetter interface {
 	UnusedIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc
 	UnusedIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc
 	UnusedIsolatedDevicesByModelAndWire(model, wire string) []*IsolatedDeviceDesc
+	UnusedIsolatedDevicesByDevicePath(devPath string) []*IsolatedDeviceDesc
 	GetIsolatedDevice(devID string) *IsolatedDeviceDesc
 	UnusedGpuDevices() []*IsolatedDeviceDesc
 	GetIsolatedDevices() []*IsolatedDeviceDesc
@@ -270,6 +271,7 @@ type IsolatedDeviceDesc struct {
 	Addr           string
 	VendorDeviceID string
 	WireId         string
+	DevicePath     string
 }
 
 func (i *IsolatedDeviceDesc) VendorID() string {

pkg/scheduler/test/mock/core.go

@@ -700,6 +700,14 @@ func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByModel(arg0 string)
 	return ret0
 }
 
+// UnusedIsolatedDevicesByDevicePath mocks base method
+func (m *MockCandidatePropertyGetter) UnusedIsolatedDevicesByDevicePath(arg0 string) []*core.IsolatedDeviceDesc {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UnusedIsolatedDevicesByDevicePath", arg0)
+	ret0, _ := ret[0].([]*core.IsolatedDeviceDesc)
+	return ret0
+}
+
 // UnusedIsolatedDevicesByModel indicates an expected call of UnusedIsolatedDevicesByModel
 func (mr *MockCandidatePropertyGetterMockRecorder) UnusedIsolatedDevicesByModel(arg0 interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()

pkg/util/pod/pod.go

@@ -33,8 +33,8 @@ type CRI interface {
 	ImageStatus(ctx context.Context, req *runtimeapi.ImageStatusRequest) (*runtimeapi.ImageStatusResponse, error)
 
 	// lower layer client
-	// getImageClient() runtimeapi.ImageServiceClient
-	// getRuntimeClient() runtimeapi.RuntimeServiceClient
+	GetImageClient() runtimeapi.ImageServiceClient
+	GetRuntimeClient() runtimeapi.RuntimeServiceClient
 }
 
 type ListContainerOptions struct {
@@ -92,23 +92,23 @@ func NewCRI(endpoint string, timeout time.Duration) (CRI, error) {
 	}, nil
 }
 
-func (c crictl) getImageClient() runtimeapi.ImageServiceClient {
+func (c crictl) GetImageClient() runtimeapi.ImageServiceClient {
 	return c.imgCli
 }
 
-func (c crictl) getRuntimeClient() runtimeapi.RuntimeServiceClient {
+func (c crictl) GetRuntimeClient() runtimeapi.RuntimeServiceClient {
 	return c.runCli
 }
 
 func (c crictl) Version(ctx context.Context) (*runtimeapi.VersionResponse, error) {
-	return c.getRuntimeClient().Version(ctx, &runtimeapi.VersionRequest{})
+	return c.GetRuntimeClient().Version(ctx, &runtimeapi.VersionRequest{})
 }
 
 func (c crictl) ListImages(ctx context.Context, filter *runtimeapi.ImageFilter) ([]*runtimeapi.Image, error) {
 	ctx, cancel := context.WithTimeout(ctx, c.timeout)
 	defer cancel()
 
-	resp, err := c.getImageClient().ListImages(ctx, &runtimeapi.ListImagesRequest{
+	resp, err := c.GetImageClient().ListImages(ctx, &runtimeapi.ListImagesRequest{
 		Filter: filter,
 	})
 	if err != nil {
@@ -123,7 +123,7 @@ func (c crictl) RunPod(ctx context.Context, podConfig *runtimeapi.PodSandboxConf
 		RuntimeHandler: runtimeHandler,
 	}
 	log.Infof("RunPodSandboxRequest: %v", req)
-	r, err := c.getRuntimeClient().RunPodSandbox(ctx, req)
+	r, err := c.GetRuntimeClient().RunPodSandbox(ctx, req)
 	if err != nil {
 		return "", errors.Wrapf(err, "RunPod with request: %s", req.String())
 	}
@@ -131,7 +131,7 @@ func (c crictl) RunPod(ctx context.Context, podConfig *runtimeapi.PodSandboxConf
 }
 
 func (c crictl) StopPod(ctx context.Context, req *runtimeapi.StopPodSandboxRequest) error {
-	_, err := c.getRuntimeClient().StopPodSandbox(ctx, req)
+	_, err := c.GetRuntimeClient().StopPodSandbox(ctx, req)
 	if err != nil {
 		return errors.Wrap(err, "StopPodSandbox")
 	}
@@ -150,7 +150,7 @@ func (c crictl) PullImageWithSandbox(ctx context.Context, image string, auth *ru
 		SandboxConfig: sandbox,
 	}
 	log.Infof("PullImageRequest: %v", req)
-	r, err := c.getImageClient().PullImage(ctx, req)
+	r, err := c.GetImageClient().PullImage(ctx, req)
 	if err != nil {
 		return nil, errors.Wrapf(err, "PullImage with %s", req)
 	}
@@ -188,7 +188,7 @@ func (c crictl) CreateContainer(ctx context.Context,
 	}
 
 	log.Infof("CreateContainerRequest: %v", req)
-	r, err := c.getRuntimeClient().CreateContainer(ctx, req)
+	r, err := c.GetRuntimeClient().CreateContainer(ctx, req)
 	if err != nil {
 		return "", errors.Wrapf(err, "CreateContainer with: %s", req)
 	}
@@ -201,7 +201,7 @@ func (c crictl) StartContainer(ctx context.Context, id string) error {
 	if id == "" {
 		return errors.Error("Id can't be empty")
 	}
-	if _, err := c.getRuntimeClient().StartContainer(ctx, &runtimeapi.StartContainerRequest{
+	if _, err := c.GetRuntimeClient().StartContainer(ctx, &runtimeapi.StartContainerRequest{
 		ContainerId: id,
 	}); err != nil {
 		return errors.Wrapf(err, "StartContainer %s", id)
@@ -270,7 +270,7 @@ func (c crictl) ListContainers(ctx context.Context, opts ListContainerOptions) (
 	req := &runtimeapi.ListContainersRequest{
 		Filter: filter,
 	}
-	r, err := c.getRuntimeClient().ListContainers(ctx, req)
+	r, err := c.GetRuntimeClient().ListContainers(ctx, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "ListContainers")
 	}
@@ -302,7 +302,7 @@ func (c crictl) ListPods(ctx context.Context, opts ListPodOptions) ([]*runtimeap
 	req := &runtimeapi.ListPodSandboxRequest{
 		Filter: filter,
 	}
-	ret, err := c.getRuntimeClient().ListPodSandbox(ctx, req)
+	ret, err := c.GetRuntimeClient().ListPodSandbox(ctx, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "ListPodSandbox")
 	}
@@ -310,7 +310,7 @@ func (c crictl) ListPods(ctx context.Context, opts ListPodOptions) ([]*runtimeap
 }
 
 func (c crictl) RemovePod(ctx context.Context, podId string) error {
-	if _, err := c.getRuntimeClient().RemovePodSandbox(ctx, &runtimeapi.RemovePodSandboxRequest{
+	if _, err := c.GetRuntimeClient().RemovePodSandbox(ctx, &runtimeapi.RemovePodSandboxRequest{
 		PodSandboxId: podId,
 	}); err != nil {
 		return errors.Wrap(err, "RemovePodSandbox")
@@ -319,7 +319,7 @@ func (c crictl) RemovePod(ctx context.Context, podId string) error {
 }
 
 func (c crictl) StopContainer(ctx context.Context, ctrId string, timeout int64) error {
-	if _, err := c.getRuntimeClient().StopContainer(ctx, &runtimeapi.StopContainerRequest{
+	if _, err := c.GetRuntimeClient().StopContainer(ctx, &runtimeapi.StopContainerRequest{
 		ContainerId: ctrId,
 		Timeout:     timeout,
 	}); err != nil {
@@ -329,7 +329,7 @@ func (c crictl) StopContainer(ctx context.Context, ctrId string, timeout int64)
 }
 
 func (c crictl) RemoveContainer(ctx context.Context, ctrId string) error {
-	_, err := c.getRuntimeClient().RemoveContainer(ctx, &runtimeapi.RemoveContainerRequest{
+	_, err := c.GetRuntimeClient().RemoveContainer(ctx, &runtimeapi.RemoveContainerRequest{
 		ContainerId: ctrId,
 	})
 	if err != nil {
@@ -343,11 +343,11 @@ func (c crictl) ContainerStatus(ctx context.Context, ctrId string) (*runtimeapi.
 		ContainerId: ctrId,
 		Verbose:     false,
 	}
-	return c.getRuntimeClient().ContainerStatus(ctx, req)
+	return c.GetRuntimeClient().ContainerStatus(ctx, req)
 }
 
 func (c crictl) PullImage(ctx context.Context, req *runtimeapi.PullImageRequest) (*runtimeapi.PullImageResponse, error) {
-	resp, err := c.getImageClient().PullImage(ctx, req)
+	resp, err := c.GetImageClient().PullImage(ctx, req)
 	if err != nil {
 		return nil, errors.Wrapf(err, "PullImage")
 	}
@@ -355,5 +355,5 @@ func (c crictl) PullImage(ctx context.Context, req *runtimeapi.PullImageRequest)
 }
 
 func (c crictl) ImageStatus(ctx context.Context, req *runtimeapi.ImageStatusRequest) (*runtimeapi.ImageStatusResponse, error) {
-	return c.getImageClient().ImageStatus(ctx, req)
+	return c.GetImageClient().ImageStatus(ctx, req)
 }