Merge pull request #17037 from wanyaoqi/feat/general-pci-device2

feat(region,host): custom pci device type

cmd/climc/shell/compute/isolated_device_models.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/compute"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.IsolatedDeviceModels)
+	cmd.List(new(options.IsolatedDeviceModelListOptions))
+	cmd.Create(new(options.IsolatedDeviceModelCreateOptions))
+	cmd.Update(new(options.IsolatedDeviceModelUpdateOptions))
+	cmd.Delete(new(options.IsolatedDeviceIdsOptions))
+}

pkg/apis/compute/guests.go

@@ -52,6 +52,10 @@ type ServerListInput struct {
 	Gpu *bool `json:"gpu"`
 	// 只列出透传了 USB 的主机
 	Usb *bool `json:"usb"`
+	// 自定义 PCI 设备类型
+	CustomDevType string `json:"custom_dev_type"`
+	// 通用虚拟机
+	Normal *bool `json:"normal"`
 	// 只列出还有备份机的主机
 	Backup *bool `json:"bakcup"`
 	// 列出指定类型的主机

pkg/apis/compute/isolated_device.go

@@ -112,3 +112,56 @@ type IsolatedDeviceJsonDesc struct {
 	DiskIndex           int8   `json:"disk_index"`
 	NvmeSizeMB          int    `json:"nvme_size_mb"`
 }
+
+type IsolatedDeviceModelCreateInput struct {
+	apis.StandaloneAnonResourceCreateInput
+
+	// 设备类型
+	// example: NPU
+	DevType string `json:"dev_type"`
+
+	// 设备型号
+	Model string `json:"model"`
+
+	// 设备VendorId
+	VendorId string `json:"vendor_id"`
+
+	// 设备DeviceId
+	DeviceId string `json:"device_id"`
+
+	// hosts scan isolated device after isolated_device_model created
+	Hosts []string `json:"hosts"`
+}
+
+type IsolatedDeviceModelUpdateInput struct {
+	apis.StandaloneAnonResourceBaseUpdateInput
+	// 设备类型
+	// example: NPU
+	DevType string `json:"dev_type"`
+
+	// 设备型号
+	Model string `json:"model"`
+
+	// 设备VendorId
+	VendorId string `json:"vendor_id"`
+
+	// 设备DeviceId
+	DeviceId string `json:"device_id"`
+}
+
+type IsolatedDeviceModelListInput struct {
+	apis.StandaloneAnonResourceListInput
+
+	// 设备类型
+	// example: NPU
+	DevType []string `json:"dev_type"`
+
+	// 设备型号
+	Model []string `json:"model"`
+
+	// 设备VendorId
+	VendorId string `json:"vendor_id"`
+
+	// 设备DeviceId
+	DeviceId string `json:"device_id"`
+}

pkg/apis/compute/isolated_device_const.go

@@ -21,15 +21,16 @@ const (
 	USB_TYPE        = "USB"
 	NIC_TYPE        = "NIC"     // nic sriov
 	NVME_PT_TYPE    = "NVME-PT" // nvme passthrough
-	NVME_TYPE       = "NVME"    // nvme sriov
 
 	NVIDIA_VENDOR_ID = "10de"
 	AMD_VENDOR_ID    = "1002"
 )
 
+const MEAT_PROBED_HOST_COUNT = "probed_host_count"
+
 var VALID_GPU_TYPES = []string{GPU_HPC_TYPE, GPU_VGA_TYPE}
 
-var VALID_PASSTHROUGH_TYPES = []string{DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE, GPU_VGA_TYPE, NVME_PT_TYPE, NVME_TYPE}
+var VALID_PASSTHROUGH_TYPES = []string{DIRECT_PCI_TYPE, USB_TYPE, NIC_TYPE, GPU_HPC_TYPE, GPU_VGA_TYPE, NVME_PT_TYPE}
 
 var ID_VENDOR_MAP = map[string]string{
 	NVIDIA_VENDOR_ID: "NVIDIA",

pkg/compute/models/capabilities.go

@@ -112,12 +112,12 @@ type SCapabilities struct {
 	ReadOnlyVpcPeerBrands         []string `json:",allowempty"`
 	ReadOnlyDisabledVpcPeerBrands []string `json:",allowempty"`
 
-	ResourceTypes      []string        `json:",allowempty"`
-	StorageTypes       []string        `json:",allowempty"` // going to remove on 2.14
-	DataStorageTypes   []string        `json:",allowempty"` // going to remove on 2.14
-	GPUModels          []string        `json:",allowempty"` // Deprecated by GPUModelTypes
-	GPUModelTypes      []GpuModelTypes `json:",allowempty"`
-	HostCpuArchs       []string        `json:",allowempty"` // x86_64 aarch64
+	ResourceTypes      []string           `json:",allowempty"`
+	StorageTypes       []string           `json:",allowempty"` // going to remove on 2.14
+	DataStorageTypes   []string           `json:",allowempty"` // going to remove on 2.14
+	GPUModels          []string           `json:",allowempty"` // Deprecated by PCIModelTypes
+	PCIModelTypes      []PCIDevModelTypes `json:",allowempty"`
+	HostCpuArchs       []string           `json:",allowempty"` // x86_64 aarch64
 	MinNicCount        int
 	MaxNicCount        int
 	MinDataDiskCount   int
@@ -196,7 +196,7 @@ func GetCapabilities(ctx context.Context, userCred mcclient.TokenCredential, que
 	capa.StorageTypes, capa.DataStorageTypes = s1, d1
 	capa.StorageTypes2, capa.StorageTypes3 = s2, s3
 	capa.DataStorageTypes2, capa.DataStorageTypes3 = d2, d3
-	capa.GPUModels, capa.GPUModelTypes = getGPUs(userCred, region, zone, domainId)
+	capa.GPUModels, capa.PCIModelTypes = getIsolatedDeviceInfo(userCred, region, zone, domainId)
 	capa.SchedPolicySupport = isSchedPolicySupported(region, zone)
 	capa.MinNicCount = getMinNicCount(region, zone)
 	capa.MaxNicCount = getMaxNicCount(region, zone)
@@ -743,13 +743,13 @@ func getStorageTypes(
 		allHypervisorStorageTypes, allHypervisorStorageInfos
 }
 
-type GpuModelTypes struct {
+type PCIDevModelTypes struct {
 	Model   string
 	DevType string
 	SizeMB  int
 }
 
-func getGPUs(userCred mcclient.TokenCredential, region *SCloudregion, zone *SZone, domainId string) ([]string, []GpuModelTypes) {
+func getIsolatedDeviceInfo(userCred mcclient.TokenCredential, region *SCloudregion, zone *SZone, domainId string) ([]string, []PCIDevModelTypes) {
 	devices := IsolatedDeviceManager.Query().SubQuery()
 	hostQuery := HostManager.Query()
 	if len(domainId) > 0 {
@@ -759,7 +759,7 @@ func getGPUs(userCred mcclient.TokenCredential, region *SCloudregion, zone *SZon
 	hosts := hostQuery.SubQuery()
 
 	q := devices.Query(devices.Field("model"), devices.Field("dev_type"), devices.Field("nvme_size_mb"))
-	q = q.Startswith("dev_type", "GPU")
+	q = q.Filter(sqlchemy.NotIn(devices.Field("dev_type"), []string{api.USB_TYPE, api.NIC_TYPE, api.NVME_PT_TYPE}))
 	if region != nil {
 		subq := getRegionZoneSubq(region)
 		q = q.Join(hosts, sqlchemy.Equals(devices.Field("host_id"), hosts.Field("id")))
@@ -777,6 +777,7 @@ func getGPUs(userCred mcclient.TokenCredential, region *SCloudregion, zone *SZon
 		))
 	}*/
 	q = q.GroupBy(devices.Field("model"), devices.Field("dev_type"), devices.Field("nvme_size_mb"))
+	q.DebugQuery()
 
 	rows, err := q.Rows()
 	if err != nil {
@@ -784,17 +785,17 @@ func getGPUs(userCred mcclient.TokenCredential, region *SCloudregion, zone *SZon
 		return nil, nil
 	}
 	defer rows.Close()
-	gpus := make([]GpuModelTypes, 0)
+	gpus := make([]PCIDevModelTypes, 0)
 	gpuModels := make([]string, 0)
 	for rows.Next() {
 		var m, t string
 		var sizeMB int
-		rows.Scan(&m, &t)
+		rows.Scan(&m, &t, &sizeMB)
 
 		if m == "" {
 			continue
 		}
-		gpus = append(gpus, GpuModelTypes{m, t, sizeMB})
+		gpus = append(gpus, PCIDevModelTypes{m, t, sizeMB})
 
 		if !utils.IsInStringArray(m, gpuModels) {
 			gpuModels = append(gpuModels, m)

pkg/compute/models/guest_actions.go

@@ -1911,8 +1911,8 @@ func (self *SGuest) startDetachIsolateDeviceWithoutNic(ctx context.Context, user
 		return httperrors.NewBadRequestError(msgFmt, device)
 	}
 	dev := iDev.(*SIsolatedDevice)
-	if dev.DevType == api.NIC_TYPE {
-		return httperrors.NewBadRequestError("Can't separately detach dev type %s", api.NIC_TYPE)
+	if dev.DevType == api.NIC_TYPE || dev.DevType == api.NVME_PT_TYPE {
+		return httperrors.NewBadRequestError("Can't separately detach dev type %s", dev.DevType)
 	}
 	if dev.IsGPU() && !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) {
 		return httperrors.NewInvalidStatusError("Can't detach GPU when status is %q", self.GetStatus())
@@ -2007,14 +2007,14 @@ func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcc
 }
 
 func (self *SGuest) StartAttachIsolatedDeviceGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error {
-	if err := self.startAttachIsolatedDevGpuOrUsb(ctx, userCred, device); err != nil {
+	if err := self.startAttachIsolatedDevGeneral(ctx, userCred, device); err != nil {
 		return err
 	}
 	// perform post attach task
 	return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "")
 }
 
-func (self *SGuest) startAttachIsolatedDevGpuOrUsb(ctx context.Context, userCred mcclient.TokenCredential, device string) error {
+func (self *SGuest) startAttachIsolatedDevGeneral(ctx context.Context, userCred mcclient.TokenCredential, device string) error {
 	iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device)
 	if err != nil {
 		msgFmt := "Isolated device %s not found"
@@ -2024,7 +2024,9 @@ func (self *SGuest) startAttachIsolatedDevGpuOrUsb(ctx context.Context, userCred
 	}
 	dev := iDev.(*SIsolatedDevice)
 	if !utils.IsInStringArray(dev.DevType, []string{api.GPU_HPC_TYPE, api.GPU_VGA_TYPE, api.USB_TYPE}) {
-		return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType)
+		if _, err = IsolatedDeviceModelManager.GetByDevType(dev.DevType); err != nil {
+			return httperrors.NewBadRequestError("Can't separately attach dev type %s", dev.DevType)
+		}
 	}
 	if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) {
 		return httperrors.NewInvalidStatusError("Can't attach GPU when status is %q", self.GetStatus())
@@ -2113,7 +2115,7 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli
 		}
 	}
 	for i := 0; i < len(addDevs); i++ {
-		err := self.startAttachIsolatedDevGpuOrUsb(ctx, userCred, addDevs[i])
+		err := self.startAttachIsolatedDevGeneral(ctx, userCred, addDevs[i])
 		if err != nil {
 			return nil, err
 		}

pkg/compute/models/guests.go

@@ -497,7 +497,7 @@ func (manager *SGuestManager) ListItemFilter(
 		var trueVal, falseVal = true, false
 		switch query.ServerType {
 		case "normal":
-			query.Gpu = &falseVal
+			query.Normal = &falseVal
 			query.Backup = &falseVal
 		case "gpu":
 			query.Gpu = &trueVal
@@ -509,7 +509,8 @@ func (manager *SGuestManager) ListItemFilter(
 			query.Usb = &trueVal
 			query.Backup = &falseVal
 		default:
-			return nil, httperrors.NewInputParameterError("unknown server type %s", query.ServerType)
+			query.CustomDevType = query.ServerType
+			query.Backup = &falseVal
 		}
 	}
 
@@ -525,10 +526,12 @@ func (manager *SGuestManager) ListItemFilter(
 		if checkType != nil {
 			conditions := []sqlchemy.ICondition{}
 			isodev := IsolatedDeviceManager.Query().SubQuery()
-			sgq := isodev.Query(isodev.Field("guest_id")).
-				Filter(sqlchemy.AND(
-					sqlchemy.IsNotNull(isodev.Field("guest_id")),
-					sqlchemy.Startswith(isodev.Field("dev_type"), dType)))
+
+			isodevCons := []sqlchemy.ICondition{sqlchemy.IsNotNull(isodev.Field("guest_id"))}
+			if len(dType) > 0 {
+				isodevCons = append(isodevCons, sqlchemy.Startswith(isodev.Field("dev_type"), dType))
+			}
+			sgq := isodev.Query(isodev.Field("guest_id")).Filter(sqlchemy.AND(isodevCons...))
 			cond := sqlchemy.NotIn
 			if *checkType {
 				cond = sqlchemy.In
@@ -543,8 +546,13 @@ func (manager *SGuestManager) ListItemFilter(
 		return q
 	}
 
+	q = devTypeQ(q, query.Normal, "")
 	q = devTypeQ(q, query.Gpu, "GPU")
 	q = devTypeQ(q, query.Usb, api.USB_TYPE)
+	if len(query.CustomDevType) > 0 {
+		ct := true
+		q = devTypeQ(q, &ct, query.CustomDevType)
+	}
 
 	groupFilter := query.GroupId
 	if len(groupFilter) != 0 {
@@ -4261,7 +4269,7 @@ func (self *SGuest) createDiskOnHost(
 
 func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error {
 	for _, devConfig := range devs {
-		if devConfig.DevType == api.NIC_TYPE {
+		if devConfig.DevType == api.NIC_TYPE || devConfig.DevType == api.NVME_PT_TYPE {
 			continue
 		}
 		err := self.createIsolatedDeviceOnHost(ctx, userCred, host, devConfig, pendingUsage)

pkg/compute/models/hosts.go

@@ -3315,6 +3315,14 @@ func (manager *SHostManager) GetHostsByManagerAndRegion(managerId string, region
 	return ret
 }
 
+func (self *SHost) RequestScanIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential) error {
+	_, err := self.Request(ctx, userCred, "POST", fmt.Sprintf("/hosts/%s/probe-isolated-devices", self.Id), mcclient.GetTokenHeaders(userCred), nil)
+	if err != nil {
+		return errors.Wrapf(err, "request host %s probe isolaed devices", self.Id)
+	}
+	return nil
+}
+
 func (self *SHost) Request(ctx context.Context, userCred mcclient.TokenCredential, method httputils.THttpMethod, url string, headers http.Header, body jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	s := auth.GetSession(ctx, userCred, "")
 	_, ret, err := s.JSONRequest(self.ManagerUri, "", method, url, headers, body)

pkg/compute/models/isolated_device_models.go

@@ -0,0 +1,239 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+var IsolatedDeviceModelManager *SIsolatedDeviceModelManager
+
+func init() {
+	IsolatedDeviceModelManager = &SIsolatedDeviceModelManager{
+		SStandaloneAnonResourceBaseManager: db.NewStandaloneAnonResourceBaseManager(
+			SIsolatedDeviceModel{},
+			"isolated_device_models_tbl",
+			"isolated_device_model",
+			"isolated_device_models",
+		),
+	}
+	IsolatedDeviceModelManager.SetVirtualObject(IsolatedDeviceModelManager)
+}
+
+type SIsolatedDeviceModelManager struct {
+	db.SStandaloneAnonResourceBaseManager
+}
+
+type SIsolatedDeviceModel struct {
+	db.SStandaloneAnonResourceBase
+
+	Model string `width:"512" charset:"ascii" nullable:"false" list:"domain" create:"domain_required" update:"domain"`
+
+	VendorId string `width:"16" charset:"ascii" nullable:"false" list:"domain" create:"domain_required" update:"domain"`
+	DeviceId string `width:"16" charset:"ascii" nullable:"false" list:"domain" create:"domain_required" update:"domain"`
+
+	DevType string `width:"16" charset:"ascii" nullable:"false" list:"domain" create:"domain_required"`
+}
+
+func (manager *SIsolatedDeviceModelManager) ValidateCreateData(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.IsolatedDeviceModelCreateInput,
+) (api.IsolatedDeviceModelCreateInput, error) {
+	if utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) {
+		return input, httperrors.NewInputParameterError("device type %q unsupported", input.DevType)
+	}
+
+	input.VendorId = strings.ToLower(input.VendorId)
+	input.DeviceId = strings.ToLower(input.DeviceId)
+	deviceVendorReg := regexp.MustCompile(`^[a-f0-9]{4}$`)
+
+	if !deviceVendorReg.MatchString(input.VendorId) {
+		return input, httperrors.NewInputParameterError("bad vendor id %s", input.VendorId)
+	}
+	if !deviceVendorReg.MatchString(input.DeviceId) {
+		return input, httperrors.NewInputParameterError("bad vendor id %s", input.DeviceId)
+	}
+
+	if cnt := manager.Query().Equals("vendor_id", input.VendorId).Equals("device_id", input.DeviceId).Count(); cnt > 0 {
+		return input, httperrors.NewDuplicateResourceError("vendor %s device %s has been registered", input.VendorId, input.DeviceId)
+	}
+
+	if cnt := manager.Query().Equals("model", input.Model).Count(); cnt > 0 {
+		return input, httperrors.NewDuplicateResourceError("model %s has been registered", input.Model)
+	}
+
+	return input, nil
+}
+
+func (self *SIsolatedDeviceModel) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
+	input := api.IsolatedDeviceModelCreateInput{}
+	err := data.Unmarshal(&input)
+	if err != nil {
+		log.Errorf("!!!data.Unmarshal api.IsolatedDeviceModelCreateInput fail %s", err)
+	}
+	go func() {
+		defer self.RemoveMetadata(ctx, api.MEAT_PROBED_HOST_COUNT, userCred)
+
+		for i := range input.Hosts {
+			iHost, err := HostManager.FetchByIdOrName(userCred, input.Hosts[i])
+			if err != nil {
+				log.Errorf("failed fetch host %s: %s", input.Hosts[i], err)
+				continue
+			}
+			host := iHost.(*SHost)
+			log.Infof("start request host %s scan isolated devices", host.GetName())
+			if err := host.RequestScanIsolatedDevices(ctx, userCred); err != nil {
+				log.Errorf("failed scan isolated device %s", err)
+			}
+			self.SetMetadata(ctx, api.MEAT_PROBED_HOST_COUNT, strconv.Itoa(i+1), userCred)
+		}
+	}()
+}
+
+func (self *SIsolatedDeviceModel) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSONObject) error {
+	modelIsEmpty, err := IsolatedDeviceManager.CheckModelIsEmpty(self.Model, self.VendorId, self.DeviceId, self.DevType)
+	if err != nil {
+		return err
+	}
+	if !modelIsEmpty {
+		return httperrors.NewNotEmptyError("device model has guests")
+	}
+	return nil
+}
+
+func (self *SIsolatedDeviceModel) PostDelete(ctx context.Context, userCred mcclient.TokenCredential) {
+	hosts, err := IsolatedDeviceManager.GetHostsByModel(self.Model, self.VendorId, self.DeviceId, self.DevType)
+	if err != nil {
+		log.Errorf("failed get hosts by isolated device model: %s", err)
+		return
+	}
+	go func() {
+		for i := range hosts {
+			iHost, err := HostManager.FetchByIdOrName(userCred, hosts[i])
+			if err != nil {
+				log.Errorf("failed fetch host %s: %s", hosts[i], err)
+				continue
+			}
+			host := iHost.(*SHost)
+			log.Infof("start request host %s scan isolated devices", host.GetName())
+			if err := host.RequestScanIsolatedDevices(ctx, userCred); err != nil {
+				log.Errorf("failed scan isolated device %s", err)
+			}
+		}
+	}()
+}
+
+func (self *SIsolatedDeviceModel) ValidateUpdateData(
+	ctx context.Context, userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject, input api.IsolatedDeviceModelUpdateInput,
+) (api.IsolatedDeviceModelUpdateInput, error) {
+	input.VendorId = strings.ToLower(input.VendorId)
+	input.DeviceId = strings.ToLower(input.DeviceId)
+	deviceVendorReg := regexp.MustCompile(`^[a-f0-9]{4}$`)
+
+	if !deviceVendorReg.MatchString(input.VendorId) {
+		return input, httperrors.NewInputParameterError("bad vendor id %s", input.VendorId)
+	}
+	if !deviceVendorReg.MatchString(input.DeviceId) {
+		return input, httperrors.NewInputParameterError("bad vendor id %s", input.DeviceId)
+	}
+
+	if self.VendorId != input.VendorId || self.DeviceId != input.DeviceId {
+		if cnt := IsolatedDeviceModelManager.Query().Equals("vendor_id", input.VendorId).Equals("device_id", input.DeviceId).Count(); cnt > 0 {
+			return input, httperrors.NewDuplicateResourceError("vendor %s device %s has been registered", input.VendorId, input.DeviceId)
+		}
+	}
+
+	if self.Model != input.Model {
+		if cnt := IsolatedDeviceModelManager.Query().Equals("model", input.Model).Count(); cnt > 0 {
+			return input, httperrors.NewDuplicateResourceError("model %s has been registered", input.Model)
+		}
+	}
+
+	return input, nil
+}
+
+func (manager *SIsolatedDeviceModelManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.IsolatedDeviceModelListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SStandaloneAnonResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneAnonResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter")
+	}
+
+	if len(query.DevType) > 0 {
+		q = q.In("dev_type", query.DevType)
+	}
+	if len(query.Model) > 0 {
+		q = q.In("model", query.Model)
+	}
+
+	if len(query.VendorId) > 0 {
+		q = q.Equals("vendor_id", query.VendorId)
+	}
+	if len(query.DeviceId) > 0 {
+		q = q.Equals("device_id", query.DeviceId)
+	}
+	return q, nil
+}
+
+func (manager *SIsolatedDeviceModelManager) GetByVendorDevice(vendorId, deviceId string) (*SIsolatedDeviceModel, error) {
+	devModel := new(SIsolatedDeviceModel)
+	err := manager.Query().Equals("vendor_id", vendorId).Equals("device_id", deviceId).First(devModel)
+	if err != nil {
+		return nil, err
+	}
+	devModel.SetModelManager(manager, devModel)
+	return devModel, nil
+}
+
+func (manager *SIsolatedDeviceModelManager) GetByDevModel(model string) (*SIsolatedDeviceModel, error) {
+	devModel := new(SIsolatedDeviceModel)
+	err := manager.Query().Equals("model", model).First(devModel)
+	if err != nil {
+		return nil, err
+	}
+	devModel.SetModelManager(manager, devModel)
+	return devModel, nil
+}
+
+func (manager *SIsolatedDeviceModelManager) GetByDevType(devType string) (*SIsolatedDeviceModel, error) {
+	devModel := new(SIsolatedDeviceModel)
+	err := manager.Query().Equals("dev_type", devType).First(devModel)
+	if err != nil {
+		return nil, err
+	}
+	devModel.SetModelManager(manager, devModel)
+	return devModel, nil
+}

pkg/compute/models/isolated_devices.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strings"
 	"time"
@@ -142,7 +143,9 @@ func (manager *SIsolatedDeviceManager) ValidateCreateData(ctx context.Context,
 		return input, httperrors.NewNotEmptyError("dev_type is empty")
 	}
 	if !utils.IsInStringArray(input.DevType, api.VALID_PASSTHROUGH_TYPES) {
-		return input, httperrors.NewInputParameterError("device type %q not supported", input.DevType)
+		if _, err := IsolatedDeviceModelManager.GetByDevType(input.DevType); err != nil {
+			return input, httperrors.NewInputParameterError("device type %q not supported", input.DevType)
+		}
 	}
 
 	input.StandaloneResourceCreateInput, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneResourceCreateInput)
@@ -199,10 +202,13 @@ func (self *SIsolatedDevice) ValidateUpdateData(
 	}
 	if input.DevType != "" && input.DevType != self.DevType {
 		if !utils.IsInStringArray(input.DevType, api.VALID_GPU_TYPES) {
-			return input, httperrors.NewInputParameterError("device type %q not support update", input.DevType)
-		}
-		if !self.IsGPU() {
-			return input, httperrors.NewInputParameterError("Can't update for device %q", self.DevType)
+			if _, err := IsolatedDeviceModelManager.GetByDevType(input.DevType); err != nil {
+				return input, httperrors.NewInputParameterError("device type %q not support update", input.DevType)
+			}
+		} else {
+			if !self.IsGPU() {
+				return input, httperrors.NewInputParameterError("Can't update for device %q", self.DevType)
+			}
 		}
 	}
 	return input, nil
@@ -894,6 +900,42 @@ func (manager *SIsolatedDeviceManager) GetDevsOnHost(hostId string, model string
 	return devs, nil
 }
 
+func (manager *SIsolatedDeviceManager) CheckModelIsEmpty(model, vendor, device, devType string) (bool, error) {
+	cnt, err := manager.Query().Equals("model", model).
+		Equals("dev_type", devType).
+		Equals("vendor_device_id", fmt.Sprintf("%s:%s", vendor, device)).
+		IsNotEmpty("guest_id").CountWithError()
+	if err != nil {
+		return false, err
+	}
+	return cnt == 0, nil
+}
+
+func (manager *SIsolatedDeviceManager) GetHostsByModel(model, vendor, device, devType string) ([]string, error) {
+	q := manager.Query("host_id").Equals("model", model).
+		Equals("dev_type", devType).
+		Equals("vendor_device_id", fmt.Sprintf("%s:%s", vendor, device)).GroupBy("host_id")
+
+	rows, err := q.Rows()
+	if err != nil && err != sql.ErrNoRows {
+		return nil, errors.Wrap(err, "q.Rows")
+	}
+	if rows == nil {
+		return nil, nil
+	}
+	defer rows.Close()
+	ret := make([]string, 0)
+	for rows.Next() {
+		var hostId string
+		err = rows.Scan(&hostId)
+		if err != nil {
+			return nil, errors.Wrap(err, "rows.Scan")
+		}
+		ret = append(ret, hostId)
+	}
+	return ret, nil
+}
+
 func (self *SIsolatedDevice) GetUniqValues() jsonutils.JSONObject {
 	return jsonutils.Marshal(map[string]string{"host_id": self.HostId})
 }

pkg/compute/service/handlers.go

@@ -128,6 +128,7 @@ func InitHandlers(app *appsrv.Application) {
 		models.ReservedipManager,
 		models.KeypairManager,
 		models.IsolatedDeviceManager,
+		models.IsolatedDeviceModelManager,
 		models.SecurityGroupManager,
 		models.SecurityGroupCacheManager,
 		models.SecurityGroupRuleManager,

pkg/hostman/hostinfo/hostinfo.go

@@ -1904,7 +1904,7 @@ func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) {
 	}
 	sriovNics := h.getNicsInterfaces(options.HostOptions.SRIOVNics)
 	if err := h.IsolatedDeviceMan.ProbePCIDevices(
-		options.HostOptions.DisableGPU, options.HostOptions.DisableUSB,
+		options.HostOptions.DisableGPU, options.HostOptions.DisableUSB, options.HostOptions.DisableCustomDevice,
 		sriovNics, offloadNics, options.HostOptions.PTNVMEConfigs,
 	); err != nil {
 		return nil, errors.Wrap(err, "ProbePCIDevices")

pkg/hostman/isolated_device/isolated_device.go

@@ -99,7 +99,7 @@ type IsolatedDeviceManager interface {
 	GetDevices() []IDevice
 	GetDeviceByIdent(vendorDevId string, addr string) IDevice
 	GetDeviceByAddr(addr string) IDevice
-	ProbePCIDevices(skipGPUs, skipUSBs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks []string) error
+	ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks []string) error
 	StartDetachTask()
 	BatchCustomProbe() error
 	AppendDetachedDevice(dev *CloudDeviceInfo)
@@ -132,7 +132,7 @@ type HostNic struct {
 	Wire      string
 }
 
-func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks []string) error {
+func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs, skipCustomDevs bool, sriovNics, ovsOffloadNics []HostNic, nvmePciDisks []string) error {
 	man.devices = make([]IDevice, 0)
 	if !skipGPUs {
 		gpus, err := getPassthroughGPUS()
@@ -147,6 +147,24 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool, sriov
 		}
 	}
 
+	if !skipCustomDevs {
+		devModels, err := man.getCustomIsolatedDeviceModels()
+		if err != nil {
+			return errors.Wrap(err, "get custom isolated device models")
+		}
+		for _, devModel := range devModels {
+			devs, err := getPassthroughPCIDevs(devModel)
+			if err != nil {
+				log.Errorf("getPassthroughPCIDevs %v: %s", devModel, err)
+				return nil
+			}
+			for i, dev := range devs {
+				man.devices = append(man.devices, dev)
+				log.Infof("Add general pci device: %d => %#v", i, dev)
+			}
+		}
+	}
+
 	if !skipUSBs {
 		usbs, err := getPassthroughUSBs()
 		if err != nil {
@@ -196,6 +214,31 @@ func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool, sriov
 	return nil
 }
 
+type IsolatedDeviceModel struct {
+	DevType  string `json:"dev_type"`
+	VendorId string `json:"vendor_id"`
+	DeviceId string `json:"device_id"`
+	Model    string `json:"model"`
+}
+
+func (man *isolatedDeviceManager) getCustomIsolatedDeviceModels() ([]IsolatedDeviceModel, error) {
+	//man.getSession().
+	params := jsonutils.NewDict()
+	params.Set("limit", jsonutils.NewInt(0))
+	params.Set("scope", jsonutils.NewString("system"))
+	res, err := modules.IsolatedDeviceModels.List(man.getSession(), jsonutils.NewDict())
+	if err != nil {
+		return nil, err
+	}
+	devModels := make([]IsolatedDeviceModel, len(res.Data))
+	for i, obj := range res.Data {
+		if err := obj.Unmarshal(&devModels[i]); err != nil {
+			return nil, errors.Wrap(err, "unmarshal isolated device model failed")
+		}
+	}
+	return devModels, nil
+}
+
 func (man *isolatedDeviceManager) getSession() *mcclient.ClientSession {
 	return man.host.GetSession()
 }
@@ -450,6 +493,33 @@ func (dev *sBaseDevice) DetectByAddr() error {
 	return nil
 }
 
+func (dev *sBaseDevice) CustomProbe(idx int) error {
+	// check environments on first probe
+	if idx == 0 {
+		for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} {
+			if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", driver).Run(); err != nil {
+				return fmt.Errorf("modprobe %s: %v", driver, err)
+			}
+		}
+	}
+
+	driver, err := dev.GetKernelDriver()
+	if err != nil {
+		return fmt.Errorf("Nic %s is occupied by another driver: %s", dev.GetAddr(), driver)
+	}
+	if driver != VFIO_PCI_KERNEL_DRIVER {
+		if driver != "" {
+			if err = dev.dev.unbindDriver(); err != nil {
+				return errors.Wrap(err, "unbind driver")
+			}
+		}
+		if err = dev.dev.bindDriver(); err != nil {
+			return errors.Wrap(err, "bind driver")
+		}
+	}
+	return nil
+}
+
 func ParseOutput(output []byte, doTrim bool) []string {
 	lines := make([]string, 0)
 	for _, line := range strings.Split(string(output), "\n") {

pkg/hostman/isolated_device/nvme.go

@@ -23,7 +23,6 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
-	"yunion.io/x/onecloud/pkg/util/procutils"
 )
 
 type sNVMEDevice struct {
@@ -115,38 +114,7 @@ func getPassthroughNVMEDisks(nvmePciDisks []string) ([]*sNVMEDevice, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "detectPCIDevByAddrWithoutIOMMUGroup")
 		}
-		driver, err := dev.getKernelDriver()
-		if err != nil {
-			return nil, err
-		}
-		if driver != VFIO_PCI_KERNEL_DRIVER {
-			if driver != "" {
-				if err = dev.unbindDriver(); err != nil {
-					return nil, err
-				}
-			}
-			if err = dev.bindDriver(); err != nil {
-				return nil, err
-			}
-		}
 		devs = append(devs, newNVMEDevice(dev, api.NVME_PT_TYPE, sizeMb))
 	}
 	return devs, nil
 }
-
-func (dev *sNVMEDevice) CustomProbe(idx int) error {
-	// check environments on first probe
-	if idx == 0 {
-		for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} {
-			if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", driver).Run(); err != nil {
-				return fmt.Errorf("modprobe %s: %v", driver, err)
-			}
-		}
-	}
-
-	driver, err := dev.GetKernelDriver()
-	if err != nil {
-		return fmt.Errorf("Nic %s is occupied by another driver: %s", dev.GetAddr(), driver)
-	}
-	return nil
-}

pkg/hostman/isolated_device/pci_device.go

@@ -0,0 +1,113 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package isolated_device
+
+import (
+	"fmt"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
+)
+
+type sGeneralPCIDevice struct {
+	*sBaseDevice
+}
+
+func (dev *sGeneralPCIDevice) GetVGACmd() string {
+	return ""
+}
+
+func (dev *sGeneralPCIDevice) GetCPUCmd() string {
+	return ""
+}
+
+func (dev *sGeneralPCIDevice) GetQemuId() string {
+	return fmt.Sprintf("dev_%s", strings.ReplaceAll(dev.GetAddr(), ":", "_"))
+}
+
+func (dev *sGeneralPCIDevice) GetHotPlugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotPlugOption, error) {
+	ret := make([]*HotPlugOption, 0)
+
+	var masterDevOpt *HotPlugOption
+	for i := 0; i < len(isolatedDev.VfioDevs); i++ {
+		cmd := isolatedDev.VfioDevs[i].HostAddr
+		if optCmd := isolatedDev.VfioDevs[i].OptionsStr(); len(optCmd) > 0 {
+			cmd += fmt.Sprintf(",%s", optCmd)
+		}
+		opts := map[string]string{
+			"host": cmd,
+			"id":   isolatedDev.VfioDevs[i].Id,
+		}
+		devOpt := &HotPlugOption{
+			Device:  isolatedDev.VfioDevs[i].DevType,
+			Options: opts,
+		}
+		if isolatedDev.VfioDevs[i].Function == 0 {
+			masterDevOpt = devOpt
+		} else {
+			ret = append(ret, devOpt)
+		}
+	}
+	// if PCI slot function 0 already assigned, qemu will reject hotplug function
+	// so put function 0 at the enda
+	if masterDevOpt == nil {
+		return nil, errors.Errorf("Device no function 0 found")
+	}
+	ret = append(ret, masterDevOpt)
+	return ret, nil
+}
+
+func (dev *sGeneralPCIDevice) GetHotUnplugOptions(isolatedDev *desc.SGuestIsolatedDevice) ([]*HotUnplugOption, error) {
+	if len(isolatedDev.VfioDevs) == 0 {
+		return nil, errors.Errorf("device %s no pci ids", isolatedDev.Id)
+	}
+
+	return []*HotUnplugOption{
+		{
+			Id: isolatedDev.VfioDevs[0].Id,
+		},
+	}, nil
+}
+
+func newGeneralPCIDevice(dev *PCIDevice, devType string) *sGeneralPCIDevice {
+	return &sGeneralPCIDevice{
+		sBaseDevice: newBaseDevice(dev, devType),
+	}
+}
+
+func getPassthroughPCIDevs(devModel IsolatedDeviceModel) ([]*sGeneralPCIDevice, error) {
+	ret, err := bashOutput(fmt.Sprintf("lspci -d %s:%s -nnmm", devModel.VendorId, devModel.DeviceId))
+	if err != nil {
+		return nil, err
+	}
+	lines := []string{}
+	for _, l := range ret {
+		if len(l) != 0 {
+			lines = append(lines, l)
+		}
+	}
+
+	devs := []*sGeneralPCIDevice{}
+	for _, line := range lines {
+		dev := NewPCIDevice2(line)
+		if dev.ModelName == "" {
+			dev.ModelName = devModel.Model
+		}
+		devs = append(devs, newGeneralPCIDevice(dev, devModel.DevType))
+	}
+	return devs, nil
+}

pkg/hostman/options/options.go

@@ -168,11 +168,12 @@ type SHostOptions struct {
 
 	DisableKVM bool `help:"force disable KVM" default:"false" json:"disable_kvm"`
 
-	DisableGPU     bool     `help:"force disable GPU detect" default:"false" json:"disable_gpu"`
-	DisableUSB     bool     `help:"force disable USB detect" default:"true" json:"disable_usb"`
-	SRIOVNics      []string `help:"nics enable sriov" json:"sriov_nics"`
-	OvsOffloadNics []string `help:"nics enable ovs offload" json:"ovs_offload_nics"`
-	PTNVMEConfigs  []string `help:"passthrough nvme disk pci address and size"`
+	DisableGPU          bool     `help:"force disable GPU detect" default:"false" json:"disable_gpu"`
+	DisableCustomDevice bool     `help:"force disable custom pci device detect" default:"false" json:"disable_custom_device"`
+	DisableUSB          bool     `help:"force disable USB detect" default:"true" json:"disable_usb"`
+	SRIOVNics           []string `help:"nics enable sriov" json:"sriov_nics"`
+	OvsOffloadNics      []string `help:"nics enable ovs offload" json:"ovs_offload_nics"`
+	PTNVMEConfigs       []string `help:"passthrough nvme disk pci address and size"`
 
 	EthtoolEnableGso bool `help:"use ethtool to turn on or off GSO(generic segment offloading)" default:"false" json:"ethtool_enable_gso"`
 

pkg/mcclient/modules/compute/mod_isolated_device_models.go

@@ -0,0 +1,32 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	IsolatedDeviceModels modulebase.ResourceManager
+)
+
+func init() {
+	IsolatedDeviceModels = modules.NewComputeManager("isolated_device_model", "isolated_device_models",
+		[]string{"ID", "Dev_type",
+			"Model", "Vendor_id", "Device_id"},
+		[]string{})
+	modules.RegisterCompute(&IsolatedDeviceModels)
+}

pkg/mcclient/options/compute/isolated_device_models.go

@@ -0,0 +1,62 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type IsolatedDeviceModelListOptions struct {
+	options.BaseListOptions
+
+	DevType  string `help:"filter by dev_type"`
+	Model    string `help:"filter by device model"`
+	VendorId string `help:"filter by vendor id"`
+	DeviceId string `help:"filter by device id"`
+}
+
+func (o *IsolatedDeviceModelListOptions) Params() (jsonutils.JSONObject, error) {
+	return options.ListStructToParams(o)
+}
+
+type IsolatedDeviceModelCreateOptions struct {
+	MODEL     string   `help:"device model name"`
+	DEV_TYPE  string   `help:"custom device type"`
+	VENDOR_ID string   `help:"pci vendor id"`
+	DEVICE_ID string   `help:"pci device id"`
+	Hosts     []string `help:"hosts id or name rescan isolated device"`
+}
+
+func (o *IsolatedDeviceModelCreateOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}
+
+type IsolatedDeviceModelUpdateOptions struct {
+	options.BaseIdOptions
+
+	MODEL     string `help:"device model name"`
+	VENDOR_ID string `help:"pci vendor id"`
+	DEVICE_ID string `help:"pci device id"`
+}
+
+func (o *IsolatedDeviceModelUpdateOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}
+
+type IsolatedDeviceIdsOptions struct {
+	options.BaseIdOptions
+}