From 8423aa9e6311cce5147fdf8d7e412205b3d2f76f Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Thu, 23 Dec 2021 20:44:50 +0800 Subject: [PATCH] feat(region,host): usb passthrough --- cmd/climc/shell/compute/hosts.go | 1 + cmd/climc/shell/compute/isolatedevices.go | 12 +- .../shell/compute/serverisolatedevices.go | 2 +- pkg/baremetal/tasks/baseprepare.go | 2 +- pkg/compute/guestdrivers/base.go | 5 + pkg/compute/guestdrivers/kvm.go | 4 + pkg/compute/hostdrivers/base.go | 4 + pkg/compute/hostdrivers/kvm.go | 8 + pkg/compute/models/guest_actions.go | 73 +- pkg/compute/models/guestdrivers.go | 2 + pkg/compute/models/hostdrivers.go | 1 + pkg/compute/models/hosts.go | 4 + pkg/compute/models/isolated_devices.go | 40 +- .../guest_attach_isolated_device_task.go | 73 ++ pkg/compute/tasks/helper.go | 21 + pkg/hostman/guestman/guesttasks.go | 142 ++++ pkg/hostman/guestman/qemu-kvm.go | 34 +- pkg/hostman/guestman/qemu-kvmhelper.go | 2 + pkg/hostman/hosthandler/handler.go | 11 +- pkg/hostman/hostinfo/hostinfo.go | 74 +- pkg/hostman/hostutils/hostutils.go | 2 +- pkg/hostman/isolated_device/gpu.go | 534 ++++++++++++++ .../isolated_device/isolated_device.go | 694 ++++-------------- pkg/hostman/isolated_device/usb.go | 212 ++++++ pkg/hostman/isolated_device/usb_test.go | 144 ++++ pkg/hostman/options/options.go | 3 +- 26 files changed, 1471 insertions(+), 633 deletions(-) create mode 100644 pkg/compute/tasks/guest_attach_isolated_device_task.go create mode 100644 pkg/compute/tasks/helper.go create mode 100644 pkg/hostman/isolated_device/gpu.go create mode 100644 pkg/hostman/isolated_device/usb.go create mode 100644 pkg/hostman/isolated_device/usb_test.go diff --git a/cmd/climc/shell/compute/hosts.go b/cmd/climc/shell/compute/hosts.go index 937c0e4195..419ca00e57 100644 --- a/cmd/climc/shell/compute/hosts.go +++ b/cmd/climc/shell/compute/hosts.go @@ -48,6 +48,7 @@ func init() { cmd.Perform("reset", &options.BaseIdOptions{}) cmd.BatchDelete(&options.BaseIdsOptions{}) cmd.Perform("remove-all-netifs", &options.BaseIdOptions{}) + cmd.Perform("probe-isolated-devices", &options.BaseIdOptions{}) cmd.BatchPerform("enable", &options.BaseIdsOptions{}) cmd.BatchPerform("disable", &options.BaseIdsOptions{}) diff --git a/cmd/climc/shell/compute/isolatedevices.go b/cmd/climc/shell/compute/isolatedevices.go index f8adf396a4..3debafaad7 100644 --- a/cmd/climc/shell/compute/isolatedevices.go +++ b/cmd/climc/shell/compute/isolatedevices.go @@ -76,12 +76,12 @@ func init() { return nil }) - R(&DeviceShowOptions{}, "isolated-device-delete", "Delete a isolated device", func(s *mcclient.ClientSession, args *DeviceShowOptions) error { - result, err := modules.IsolatedDevices.Delete(s, args.ID, nil) - if err != nil { - return err - } - printObject(result) + type DeviceDeleteOptions struct { + IDS []string `help:"IDs of the isolated device"` + } + R(&DeviceDeleteOptions{}, "isolated-device-delete", "Delete a isolated device", func(s *mcclient.ClientSession, args *DeviceDeleteOptions) error { + result := modules.IsolatedDevices.BatchDelete(s, args.IDS, nil) + printBatchResults(result, modules.IsolatedDevices.GetColumns(s)) return nil }) diff --git a/cmd/climc/shell/compute/serverisolatedevices.go b/cmd/climc/shell/compute/serverisolatedevices.go index 23b1efc7f6..1b55f99562 100644 --- a/cmd/climc/shell/compute/serverisolatedevices.go +++ b/cmd/climc/shell/compute/serverisolatedevices.go @@ -48,7 +48,7 @@ func init() { R(&ServerDetachDeviceOptions{}, "server-detach-isolated-device", "Detach a isolated device from a virtual server", func(s *mcclient.ClientSession, args *ServerDetachDeviceOptions) error { params := jsonutils.NewDict() params.Add(jsonutils.NewString(args.DEVICE), "device") - srv, err := modules.Servers.PerformAction(s, args.SERVER, "detach-isolated_device", params) + srv, err := modules.Servers.PerformAction(s, args.SERVER, "detach-isolated-device", params) if err != nil { return err } diff --git a/pkg/baremetal/tasks/baseprepare.go b/pkg/baremetal/tasks/baseprepare.go index e29bc55bba..dff5bbc759 100644 --- a/pkg/baremetal/tasks/baseprepare.go +++ b/pkg/baremetal/tasks/baseprepare.go @@ -834,7 +834,7 @@ func (task *sBaremetalPrepareTask) sendIsolatedDevicesInfo( } for i := 0; i < len(gpuDevs); i++ { - if err := gpuDevs[i].SyncDeviceInfo(session, task.baremetal.GetId()); err != nil { + if _, err := isolated_device.SyncDeviceInfo(session, task.baremetal.GetId(), gpuDevs[i]); err != nil { return errors.Wrap(err, "sync device info") } } diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 2e69e6ae35..f7462f587d 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -448,3 +448,8 @@ func (self *SBaseGuestDriver) StartChangeDiskStorageTask(guest *models.SGuest, c func (self *SBaseGuestDriver) RequestChangeDiskStorage(ctx context.Context, userCred mcclient.TokenCredential, guest *models.SGuest, input *api.ServerChangeDiskStorageInternalInput, task taskman.ITask) error { return cloudprovider.ErrNotImplemented } + +func (self *SBaseGuestDriver) RequestSyncIsolatedDevice(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + task.ScheduleRun(nil) + return nil +} diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 8235dc6c5f..d02855e90e 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -913,3 +913,7 @@ func (self *SKVMGuestDriver) ValidateUpdateData(ctx context.Context, guest *mode return input, nil } + +func (self *SKVMGuestDriver) RequestSyncIsolatedDevice(ctx context.Context, guest *models.SGuest, task taskman.ITask) error { + return guest.StartSyncTask(ctx, task.GetUserCred(), false, task.GetTaskId()) +} diff --git a/pkg/compute/hostdrivers/base.go b/pkg/compute/hostdrivers/base.go index d5565b18f8..5a48fd5902 100644 --- a/pkg/compute/hostdrivers/base.go +++ b/pkg/compute/hostdrivers/base.go @@ -265,3 +265,7 @@ func (driver *SBaseHostDriver) RequestDeallocateBackupDiskOnHost(ctx context.Con func (driver *SBaseHostDriver) RequestSyncOnHost(ctx context.Context, host *models.SHost, task taskman.ITask) error { return nil } + +func (driver *SBaseHostDriver) RequestProbeIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, input jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return nil, nil +} diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go index ef1e93c24f..22d98ec2a0 100644 --- a/pkg/compute/hostdrivers/kvm.go +++ b/pkg/compute/hostdrivers/kvm.go @@ -604,3 +604,11 @@ func (self *SKVMHostDriver) GetJsonFromHost(ctx context.Context, host *models.SH } return desc } + +func (driver *SKVMHostDriver) RequestProbeIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, input jsonutils.JSONObject) (jsonutils.JSONObject, error) { + url := fmt.Sprintf("%s/hosts/%s/probe-isolated-devices", host.ManagerUri, host.GetId()) + httpClient := httputils.GetDefaultClient() + header := mcclient.GetTokenHeaders(userCred) + _, respBody, err := httputils.JSONRequest(httpClient, ctx, "POST", url, header, input, false) + return respBody, err +} diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 9110322aec..d6dd67756d 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -763,7 +763,6 @@ func (self *SGuest) StartSyncTaskWithoutSyncstatus(ctx context.Context, userCred func (self *SGuest) doSyncTask(ctx context.Context, data *jsonutils.JSONDict, userCred mcclient.TokenCredential, parentTaskId string) error { if task, err := taskman.TaskManager.NewTask(ctx, "GuestSyncConfTask", self, userCred, data, parentTaskId, "", nil); err != nil { - log.Errorln(err) return err } else { task.ScheduleRun(nil) @@ -1741,8 +1740,8 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc if self.Hypervisor != api.HYPERVISOR_KVM { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } - if self.Status != api.VM_READY { - msg := "Only allowed to attach isolated device when guest is ready" + if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { + msg := fmt.Sprintf("Can't detach isolated device when guest is %s", self.GetStatus()) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_DETACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewInvalidStatusError(msg) } @@ -1770,10 +1769,7 @@ func (self *SGuest) PerformDetachIsolatedDevice(ctx context.Context, userCred mc } } } - if jsonutils.QueryBoolean(data, "auto_start", false) { - return self.PerformStart(ctx, userCred, query, data) - } - return nil, nil + return nil, self.startIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") } func (self *SGuest) startDetachIsolateDevice(ctx context.Context, userCred mcclient.TokenCredential, device string) error { @@ -1785,6 +1781,9 @@ func (self *SGuest) startDetachIsolateDevice(ctx context.Context, userCred mccli return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) + if dev.IsGPU() && self.GetStatus() != api.VM_READY { + return httperrors.NewInvalidStatusError("Can't detach GPU when status is %q", self.GetStatus()) + } host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) @@ -1813,15 +1812,16 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc if self.Hypervisor != api.HYPERVISOR_KVM { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } - if self.Status != api.VM_READY { - msg := "Only allowed to attach isolated device when guest is ready" + if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { + msg := fmt.Sprintf("Can't attach isolated device when guest is %s", self.GetStatus()) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_GUEST_ATTACH_ISOLATED_DEVICE, msg, userCred, false) return nil, httperrors.NewInvalidStatusError(msg) } var err error + autoStart := jsonutils.QueryBoolean(data, "auto_start", false) if data.Contains("device") { device, _ := data.GetString("device") - err = self.startAttachIsolatedDevice(ctx, userCred, device) + err = self.StartAttachIsolatedDevice(ctx, userCred, device, autoStart) } else if data.Contains("model") { vmodel, _ := data.GetString("model") var count int64 = 1 @@ -1831,7 +1831,7 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc if count < 1 { return nil, httperrors.NewBadRequestError("guest attach gpu count must > 0") } - err = self.startAttachIsolatedDevices(ctx, userCred, vmodel, int(count)) + err = self.StartAttachIsolatedDevices(ctx, userCred, vmodel, int(count), autoStart) } else { return nil, httperrors.NewMissingParameterError("device||model") } @@ -1839,17 +1839,22 @@ func (self *SGuest) PerformAttachIsolatedDevice(ctx context.Context, userCred mc if err != nil { return nil, err } - if jsonutils.QueryBoolean(data, "auto_start", false) { - return self.PerformStart(ctx, userCred, query, data) - } return nil, nil } -func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, gpuModel string, count int) error { +func (self *SGuest) StartAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int, autoStart bool) error { + if err := self.startAttachIsolatedDevices(ctx, userCred, devModel, count); err != nil { + return err + } + // perform post attach task + return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") +} + +func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, devModel string, count int) error { host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) - devs, err := IsolatedDeviceManager.GetDevsOnHost(host.Id, gpuModel, count) + devs, err := IsolatedDeviceManager.GetDevsOnHost(host.Id, devModel, count) if err != nil { return httperrors.NewInternalServerError("fetch gpu failed %s", err) } @@ -1866,6 +1871,14 @@ func (self *SGuest) startAttachIsolatedDevices(ctx context.Context, userCred mcc return nil } +func (self *SGuest) StartAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, device string, autoStart bool) error { + if err := self.startAttachIsolatedDevice(ctx, userCred, device); err != nil { + return err + } + // perform post attach task + return self.startIsolatedDevicesSyncTask(ctx, userCred, autoStart, "") +} + func (self *SGuest) startAttachIsolatedDevice(ctx context.Context, userCred mcclient.TokenCredential, device string) error { iDev, err := IsolatedDeviceManager.FetchByIdOrName(userCred, device) if err != nil { @@ -1875,6 +1888,9 @@ func (self *SGuest) startAttachIsolatedDevice(ctx context.Context, userCred mccl return httperrors.NewBadRequestError(msgFmt, device) } dev := iDev.(*SIsolatedDevice) + if dev.IsGPU() && self.GetStatus() != api.VM_READY { + return httperrors.NewInvalidStatusError("Can't attach GPU when status is %q", self.GetStatus()) + } host, _ := self.GetHost() lockman.LockObject(ctx, host) defer lockman.ReleaseObject(ctx, host) @@ -1893,7 +1909,9 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient. if len(dev.GuestId) > 0 { return fmt.Errorf("Isolated device already attached to another guest: %s", dev.GuestId) } - if dev.HostId != self.HostId { + if dev.HostId != + + self.HostId { return fmt.Errorf("Isolated device and guest are not located in the same host") } _, err := db.Update(dev, func() error { @@ -1911,8 +1929,8 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli if self.Hypervisor != api.HYPERVISOR_KVM { return nil, httperrors.NewNotAcceptableError("Not allow for hypervisor %s", self.Hypervisor) } - if self.Status != api.VM_READY { - return nil, httperrors.NewInvalidStatusError("Only allowed to attach isolated device when guest is ready") + if !utils.IsInStringArray(self.GetStatus(), []string{api.VM_READY, api.VM_RUNNING}) { + return nil, httperrors.NewInvalidStatusError("Can't set isolated device when guest is %s", self.GetStatus()) } var addDevs []string { @@ -1953,10 +1971,21 @@ func (self *SGuest) PerformSetIsolatedDevice(ctx context.Context, userCred mccli return nil, err } } - if jsonutils.QueryBoolean(data, "auto_start", false) { - return self.PerformStart(ctx, userCred, query, data) + return nil, self.startIsolatedDevicesSyncTask(ctx, userCred, jsonutils.QueryBoolean(data, "auto_start", false), "") +} + +func (self *SGuest) startIsolatedDevicesSyncTask(ctx context.Context, userCred mcclient.TokenCredential, autoStart bool, parenetId string) error { + if self.GetStatus() == api.VM_RUNNING { + autoStart = false + } + data := jsonutils.Marshal(map[string]interface{}{ + "auto_start": autoStart, + }).(*jsonutils.JSONDict) + if task, err := taskman.TaskManager.NewTask(ctx, "GuestIsolatedDeviceSyncTask", self, userCred, data, parenetId, "", nil); err != nil { + return err + } else { + return task.ScheduleRun(nil) } - return nil, nil } func (self *SGuest) findGuestnetworkByInfo(ipStr string, macStr string, index int64) (*SGuestnetwork, error) { diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index dd6587434f..f64909304d 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -215,6 +215,8 @@ type IGuestDriver interface { ValidateChangeDiskStorage(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerChangeDiskStorageInput) error StartChangeDiskStorageTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *api.ServerChangeDiskStorageInternalInput, parentTaskId string) error RequestChangeDiskStorage(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, input *api.ServerChangeDiskStorageInternalInput, task taskman.ITask) error + + RequestSyncIsolatedDevice(ctx context.Context, guest *SGuest, task taskman.ITask) error } var guestDrivers map[string]IGuestDriver diff --git a/pkg/compute/models/hostdrivers.go b/pkg/compute/models/hostdrivers.go index 33a756aa35..bb36833268 100644 --- a/pkg/compute/models/hostdrivers.go +++ b/pkg/compute/models/hostdrivers.go @@ -63,6 +63,7 @@ type IHostDriver interface { RequestAttachStorage(ctx context.Context, hoststorage *SHoststorage, host *SHost, storage *SStorage, task taskman.ITask) error RequestDetachStorage(ctx context.Context, host *SHost, storage *SStorage, task taskman.ITask) error RequestSyncOnHost(ctx context.Context, host *SHost, task taskman.ITask) error + RequestProbeIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, input jsonutils.JSONObject) (jsonutils.JSONObject, error) } var hostDrivers map[string]IHostDriver diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index f168bfaa9a..c86ba86778 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -5806,3 +5806,7 @@ func (manager *SHostManager) initHostname() error { func (manager *SHostManager) InitializeData() error { return manager.initHostname() } + +func (self *SHost) PerformProbeIsolatedDevices(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return self.GetHostDriver().RequestProbeIsolatedDevices(ctx, userCred, self, data) +} diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 1c5d8af1f2..41b2104ef7 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -95,14 +95,14 @@ type SIsolatedDevice struct { VendorDeviceId string `width:"16" charset:"ascii" nullable:"true" list:"domain" create:"domain_optional"` - // reserved memory size for isolated device, default 8G - ReservedMemory int `nullable:"true" default:"8192" list:"domain" update:"domain" create:"domain_optional"` + // reserved memory size for isolated device + ReservedMemory int `nullable:"true" efault:"0" list:"domain" update:"domain" create:"domain_optional"` - // reserved cpu count for isolated device, default 8 - ReservedCpu int `nullable:"true" default:"8" list:"domain" update:"domain" create:"domain_optional"` + // reserved cpu count for isolated device + ReservedCpu int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"` - // reserved storage size for isolated device, default 100G - ReservedStorage int `nullable:"true" default:"102400" list:"domain" update:"domain" create:"domain_optional"` + // reserved storage size for isolated device + ReservedStorage int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"` } func (manager *SIsolatedDeviceManager) ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition { @@ -126,11 +126,35 @@ func (manager *SIsolatedDeviceManager) ValidateCreateData(ctx context.Context, input.Name = fmt.Sprintf("dev_%s_%d", host.GetName(), time.Now().UnixNano()) } + // validate DevType + if input.DevType == "" { + return input, httperrors.NewNotEmptyError("dev_type is empty") + } + if !utils.IsInStringArray(input.DevType, []string{api.GPU_HPC_TYPE, api.GPU_VGA_TYPE, api.USB_TYPE, api.NIC_TYPE}) { + return input, httperrors.NewInputParameterError("device type %q not supported", input.DevType) + } + input.StandaloneResourceCreateInput, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneResourceCreateInput) if err != nil { return input, errors.Wrap(err, "SStandaloneResourceBaseManager.ValidateCreateData") } + // validate reserverd resource + // inject default reserverd resource for gpu: + if utils.IsInStringArray(input.DevType, []string{api.GPU_HPC_TYPE, api.GPU_VGA_TYPE}) { + defaultCPU := 8 // 8 + defaultMem := 8192 // 8g + defaultStore := 102400 // 100g + if input.ReservedCpu == nil { + input.ReservedCpu = &defaultCPU + } + if input.ReservedMemory == nil { + input.ReservedMemory = &defaultMem + } + if input.ReservedStorage == nil { + input.ReservedStorage = &defaultStore + } + } if input.ReservedCpu != nil && *input.ReservedCpu < 0 { return input, httperrors.NewInputParameterError("reserved cpu must >= 0") } @@ -331,7 +355,7 @@ func (self *SIsolatedDevice) getVendor() string { } } -func (self *SIsolatedDevice) isGpu() bool { +func (self *SIsolatedDevice) IsGPU() bool { return strings.HasPrefix(self.DevType, "GPU") } @@ -370,7 +394,7 @@ func (manager *SIsolatedDeviceManager) parseDeviceInfo(userCred mcclient.TokenCr devConfig.Model = dev.Model devConfig.DevType = dev.DevType devConfig.Vendor = dev.getVendor() - if dev.isGpu() && len(devType) > 0 { + if dev.IsGPU() && len(devType) > 0 { if !utils.IsInStringArray(devType, VALID_GPU_TYPES) { return nil, fmt.Errorf("%s not valid for GPU device", devType) } diff --git a/pkg/compute/tasks/guest_attach_isolated_device_task.go b/pkg/compute/tasks/guest_attach_isolated_device_task.go new file mode 100644 index 0000000000..7fa9099463 --- /dev/null +++ b/pkg/compute/tasks/guest_attach_isolated_device_task.go @@ -0,0 +1,73 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type GuestIsolatedDeviceSyncTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestIsolatedDeviceSyncTask{}) +} + +func (self *GuestIsolatedDeviceSyncTask) needStart() bool { + return jsonutils.QueryBoolean(self.Params, "auto_start", false) +} + +func (self *GuestIsolatedDeviceSyncTask) onTaskFail(ctx context.Context, guest *models.SGuest, err jsonutils.JSONObject) { + self.SetStageFailed(ctx, err) + logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_SYNC_CONF, err, self.GetUserCred(), false) +} + +func (self *GuestIsolatedDeviceSyncTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + self.SetStage("OnSyncConfigComplete", nil) + if err := guest.GetDriver().RequestSyncIsolatedDevice(ctx, guest, self); err != nil { + self.onTaskFail(ctx, guest, jsonErrorObj(err)) + return + } +} + +func (self *GuestIsolatedDeviceSyncTask) OnSyncConfigComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { + if self.needStart() { + self.SetStage("OnStartComplete", nil) + guest.StartGueststartTask(ctx, self.GetUserCred(), nil, self.GetId()) + } else { + self.OnStartComplete(ctx, guest, data) + } +} + +func (self *GuestIsolatedDeviceSyncTask) OnSyncConfigCompleteFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) { + self.onTaskFail(ctx, guest, reason) +} + +func (self *GuestIsolatedDeviceSyncTask) OnStartComplete(ctx context.Context, obj *models.SGuest, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func (self *GuestIsolatedDeviceSyncTask) OnStartCompleteFailed(ctx context.Context, obj *models.SGuest, data jsonutils.JSONObject) { + self.onTaskFail(ctx, obj, data) +} diff --git a/pkg/compute/tasks/helper.go b/pkg/compute/tasks/helper.go new file mode 100644 index 0000000000..573773d46e --- /dev/null +++ b/pkg/compute/tasks/helper.go @@ -0,0 +1,21 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tasks + +import "yunion.io/x/jsonutils" + +func jsonErrorObj(err error) jsonutils.JSONObject { + return jsonutils.NewString(err.Error()) +} diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 57f1d3758d..9e6e9bf145 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -29,7 +29,9 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/appctx" + "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/hostutils" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" "yunion.io/x/onecloud/pkg/util/fileutils2" @@ -465,6 +467,146 @@ func NewGuestNetworkSyncTask(guest *SKVMGuestInstance, delNics, addNics []jsonut return &SGuestNetworkSyncTask{guest, delNics, addNics, make([]error, 0), nil} } +/** + * GuestIsolatedDeviceSyncTask +**/ + +type SGuestIsolatedDeviceSyncTask struct { + guest *SKVMGuestInstance + delDevs []jsonutils.JSONObject + addDevs []jsonutils.JSONObject + errors []error + + callback func(...error) +} + +func NewGuestIsolatedDeviceSyncTask(guest *SKVMGuestInstance, delDevs, addDevs []jsonutils.JSONObject) *SGuestIsolatedDeviceSyncTask { + return &SGuestIsolatedDeviceSyncTask{guest, delDevs, addDevs, make([]error, 0), nil} +} + +func (t *SGuestIsolatedDeviceSyncTask) Start(cb func(...error)) { + t.callback = cb + t.syncDevice() +} + +func (t *SGuestIsolatedDeviceSyncTask) syncDevice() { + if len(t.delDevs) > 0 { + dev := t.delDevs[len(t.delDevs)-1] + t.delDevs = t.delDevs[:len(t.delDevs)-1] + t.removeDevice(dev) + } else if len(t.addDevs) > 0 { + dev := t.addDevs[len(t.addDevs)-1] + t.addDevs = t.addDevs[:len(t.addDevs)-1] + t.addDevice(dev) + } else { + t.callback(t.errors...) + } +} + +func (t *SGuestIsolatedDeviceSyncTask) removeDevice(dev jsonutils.JSONObject) { + cb := func(res string) { + if len(res) > 0 { + t.errors = append(t.errors, fmt.Errorf("device del failed: %s", res)) + } else { + t.syncDevice() + } + } + + vendorDevId, err := dev.GetString("vendor_device_id") + if err != nil { + cb(err.Error()) + return + } + addr, err := dev.GetString("addr") + if err != nil { + cb(err.Error()) + return + } + + devObj := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByIdent(vendorDevId, addr) + if devObj == nil { + cb(fmt.Sprintf("Not found host isolated_device by %s %s", vendorDevId, addr)) + return + } + + opts, err := devObj.GetHotUnplugOptions() + if err != nil { + cb(errors.Wrap(err, "GetHotPlugOptions").Error()) + return + } + + t.delDeviceCallBack(opts, 0, cb) +} + +func (t *SGuestIsolatedDeviceSyncTask) addDevice(dev jsonutils.JSONObject) { + cb := func(res string) { + if len(res) > 0 { + t.errors = append(t.errors, fmt.Errorf("device add failed: %s", res)) + } else { + t.syncDevice() + } + } + + vendorDevId, err := dev.GetString("vendor_device_id") + if err != nil { + cb(err.Error()) + return + } + addr, err := dev.GetString("addr") + if err != nil { + cb(err.Error()) + return + } + + devObj := hostinfo.Instance().IsolatedDeviceMan.GetDeviceByIdent(vendorDevId, addr) + if devObj == nil { + cb(fmt.Sprintf("Not found host isolated_device by %s %s", vendorDevId, addr)) + return + } + + opts, err := devObj.GetHotPlugOptions() + if err != nil { + cb(errors.Wrap(err, "GetHotPlugOptions").Error()) + return + } + + // TODO: support GPU + t.addDeviceCallBack(opts, 0, cb) +} + +func (t *SGuestIsolatedDeviceSyncTask) addDeviceCallBack(opts []*isolated_device.HotPlugOption, idx int, onAddFinish func(string)) { + if idx >= len(opts) { + onAddFinish("") + return + } + + opt := opts[idx] + t.guest.Monitor.DeviceAdd(opt.Device, opt.Options, func(err string) { + if err != "" { + onAddFinish(fmt.Sprintf("monitor add %d device: %s", idx, err)) + return + } + t.addDeviceCallBack(opts, idx+1, onAddFinish) + }) +} + +func (t *SGuestIsolatedDeviceSyncTask) delDeviceCallBack(opts []*isolated_device.HotUnplugOption, idx int, onDelFinish func(string)) { + if idx >= len(opts) { + onDelFinish("") + return + } + + opt := opts[idx] + + t.guest.Monitor.DeviceDel(opt.Id, func(err string) { + if err != "" { + onDelFinish(fmt.Sprintf("monitor del %d device: %s", idx, err)) + return + } + t.delDeviceCallBack(opts, idx+1, onDelFinish) + }) +} + /** * GuestLiveMigrateTask **/ diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 4852bfe730..8ffd4a2d6b 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -1081,6 +1081,31 @@ func (s *SKVMGuestInstance) compareDescDisks(newDesc jsonutils.JSONObject) ([]js return delDisks, addDisks } +func (s *SKVMGuestInstance) compareDescIsolatedDevices(newDesc jsonutils.JSONObject) ([]jsonutils.JSONObject, []jsonutils.JSONObject) { + var delDevs, addDevs = []jsonutils.JSONObject{}, []jsonutils.JSONObject{} + newDevs, _ := newDesc.GetArray("isolated_devices") + for _, dev := range newDevs { + addDevs = append(addDevs, dev) + } + oldDevs, _ := s.Desc.GetArray("isolated_devices") + for _, oldDev := range oldDevs { + var find = false + oVendorDevId, _ := oldDev.GetString("vendor_device_id") + for idx, addDev := range addDevs { + nVendorDevId, _ := addDev.GetString("vendor_device_id") + if oVendorDevId == nVendorDevId { + addDevs = append(addDevs[:idx], addDevs[idx+1:]...) + find = true + break + } + } + if !find { + delDevs = append(delDevs, oldDev) + } + } + return delDevs, addDevs +} + func (s *SKVMGuestInstance) compareDescCdrom(newDesc jsonutils.JSONObject) *string { if !s.Desc.Contains("cdrom") && !newDesc.Contains("cdrom") { return nil @@ -1211,7 +1236,7 @@ func onNicChange(oldNic, newNic jsonutils.JSONObject) error { } func (s *SKVMGuestInstance) SyncConfig(ctx context.Context, desc jsonutils.JSONObject, fwOnly bool) (jsonutils.JSONObject, error) { - var delDisks, addDisks, delNetworks, addNetworks []jsonutils.JSONObject + var delDisks, addDisks, delNetworks, addNetworks, delDevs, addDevs []jsonutils.JSONObject var changedNetworks [][]jsonutils.JSONObject var cdrom *string @@ -1219,6 +1244,7 @@ func (s *SKVMGuestInstance) SyncConfig(ctx context.Context, desc jsonutils.JSONO delDisks, addDisks = s.compareDescDisks(desc) cdrom = s.compareDescCdrom(desc) delNetworks, addNetworks, changedNetworks = s.compareDescNetworks(desc) + delDevs, addDevs = s.compareDescIsolatedDevices(desc) } if len(changedNetworks) > 0 && s.IsRunning() { @@ -1278,6 +1304,12 @@ func (s *SKVMGuestInstance) SyncConfig(ctx context.Context, desc jsonutils.JSONO tasks = append(tasks, task) } + if len(delDevs)+len(addDevs) > 0 { + task := NewGuestIsolatedDeviceSyncTask(s, delDevs, addDevs) + runTaskNames = append(runTaskNames, jsonutils.NewString("isolated_device_sync")) + tasks = append(tasks, task) + } + NewGuestSyncConfigTaskExecutor(ctx, s, tasks, callBack).Start(1) res := jsonutils.NewDict() res.Set("task", jsonutils.NewArray(runTaskNames...)) diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 5713d971ec..2c0ec44176 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -683,6 +683,8 @@ function nic_mtu() { cmd += s.getVnicDesc(nics[i], true) } + // USB 3.0 + cmd += " -device qemu-xhci,id=usb" if isolatedDevsParams != nil { for _, each := range isolatedDevsParams.Devices { cmd += each diff --git a/pkg/hostman/hosthandler/handler.go b/pkg/hostman/hosthandler/handler.go index 6ddae50ef5..f2cc4d2fa0 100644 --- a/pkg/hostman/hosthandler/handler.go +++ b/pkg/hostman/hosthandler/handler.go @@ -40,7 +40,8 @@ func AddHostHandler(prefix string, app *appsrv.Application) { auth.Authenticate(setOnHostDown)) for action, f := range map[string]actionFunc{ - "sync": hostSync, + "sync": hostSync, + "probe-isolated-devices": hostProbeIsolatedDevices, } { app.AddHandler("POST", fmt.Sprintf("%s/%s//%s", prefix, keyword, action), @@ -76,6 +77,10 @@ func hostActions(f actionFunc) appsrv.FilterHandler { } } -func hostSync(ctx context.Context, sid string, body jsonutils.JSONObject) (interface{}, error) { - return hostinfo.Instance().UpdateSyncInfo(sid, body) +func hostSync(ctx context.Context, hostId string, body jsonutils.JSONObject) (interface{}, error) { + return hostinfo.Instance().UpdateSyncInfo(hostId, body) +} + +func hostProbeIsolatedDevices(ctx context.Context, hostId string, body jsonutils.JSONObject) (interface{}, error) { + return hostinfo.Instance().ProbeSyncIsolatedDevices(hostId, body) } diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 424852e0c1..6f9976096c 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -79,7 +79,7 @@ type SHostInfo struct { enableHugePages bool onHostDown string - IsolatedDeviceMan *isolated_device.IsolatedDeviceManager + IsolatedDeviceMan isolated_device.IsolatedDeviceManager MasterNic *netutils2.SNetInterface Nics []*SNIC @@ -100,7 +100,7 @@ type SHostInfo struct { IoScheduler string } -func (h *SHostInfo) GetIsolatedDeviceManager() *isolated_device.IsolatedDeviceManager { +func (h *SHostInfo) GetIsolatedDeviceManager() isolated_device.IsolatedDeviceManager { return h.IsolatedDeviceMan } @@ -1001,6 +1001,13 @@ func (h *SHostInfo) UpdateSyncInfo(hostId string, body jsonutils.JSONObject) (in return nil, nil } +func (h *SHostInfo) ProbeSyncIsolatedDevices(hostId string, body jsonutils.JSONObject) (interface{}, error) { + if h.GetHostId() != hostId { + return nil, nil + } + return h.probeSyncIsolatedDevices() +} + func (h *SHostInfo) setHostname(name string) { h.FullName = name err := sysutils.SetHostname(name) @@ -1148,7 +1155,7 @@ func (h *SHostInfo) onUpdateHostInfoSucc(hostbody jsonutils.JSONObject) { } if options.HostOptions.HugepagesOption == "native" { - if h.isInit && len(h.IsolatedDeviceMan.Devices) > 0 { + if h.isInit && len(h.IsolatedDeviceMan.GetDevices()) > 0 { meta := jsonutils.NewDict() meta.Set("__enable_hugepages", jsonutils.NewString("true")) _, err := modules.Hosts.SetMetadata(h.GetSession(), h.HostId, meta) @@ -1504,16 +1511,7 @@ func (h *SHostInfo) uploadStorageInfo() { go storageman.StartSyncStorageSizeTask( time.Duration(options.HostOptions.SyncStorageInfoDurationSecond) * time.Second, ) - var err error - if !options.HostOptions.DisableGPU { - err = h.IsolatedDeviceMan.ProbePCIDevices() - } - if err != nil { - h.onFail(errors.Wrap(err, "Probe PCI device failed")) - return - } else { - h.getIsolatedDevices() - } + h.probeSyncIsolatedDevicesStep() } func (h *SHostInfo) onSyncStorageInfoSucc(storage storageman.IStorage, storageInfo jsonutils.JSONObject) { @@ -1541,7 +1539,7 @@ func (h *SHostInfo) attachStorage(storage storageman.IStorage) { } } -func (h *SHostInfo) getIsolatedDevices() { +func (h *SHostInfo) getRemoteIsolatedDevices() ([]jsonutils.JSONObject, error) { params := jsonutils.NewDict() params.Set("details", jsonutils.JSONTrue) params.Set("limit", jsonutils.NewInt(0)) @@ -1549,18 +1547,34 @@ func (h *SHostInfo) getIsolatedDevices() { params.Set("scope", jsonutils.NewString("system")) res, err := modules.IsolatedDevices.List(h.GetSession(), params) if err != nil { - h.onFail(fmt.Sprintf("getIsolatedDevices: %v", err)) - return + return nil, err } - h.onGetIsolatedDeviceSucc(res.Data) + return res.Data, nil } -func (h *SHostInfo) onGetIsolatedDeviceSucc(objs []jsonutils.JSONObject) { +func (h *SHostInfo) probeSyncIsolatedDevicesStep() { + _, err := h.probeSyncIsolatedDevices() + if err != nil { + h.onFail(errors.Wrap(err, "probeSyncIsolatedDevices")) + return + } + + h.deployAdminAuthorizedKeys() +} + +func (h *SHostInfo) probeSyncIsolatedDevices() (*jsonutils.JSONArray, error) { + if err := h.IsolatedDeviceMan.ProbePCIDevices(options.HostOptions.DisableGPU, options.HostOptions.DisableUSB); err != nil { + return nil, errors.Wrap(err, "ProbePCIDevices") + } + + objs, err := h.getRemoteIsolatedDevices() + if err != nil { + return nil, errors.Wrap(err, "getRemoteIsolatedDevices") + } for _, obj := range objs { info := isolated_device.CloudDeviceInfo{} if err := obj.Unmarshal(&info); err != nil { - h.onFail(fmt.Sprintf("unmarshal isolated device to cloud device info failed %s", err)) - return + return nil, errors.Wrapf(err, "unmarshal isolated device %s to cloud device info", obj) } dev := h.IsolatedDeviceMan.GetDeviceByIdent(info.VendorDeviceId, info.Addr) if dev != nil { @@ -1572,21 +1586,19 @@ func (h *SHostInfo) onGetIsolatedDeviceSucc(objs []jsonutils.JSONObject) { } h.IsolatedDeviceMan.StartDetachTask() if err := h.IsolatedDeviceMan.BatchCustomProbe(); err != nil { - h.onFail(fmt.Sprintf("Device probe error: %v", err)) - return + return nil, errors.Wrap(err, "Device probe") } - h.uploadIsolatedDevices() -} -func (h *SHostInfo) uploadIsolatedDevices() { - for _, dev := range h.IsolatedDeviceMan.Devices { - if err := dev.SyncDeviceInfo(h.GetSession(), h.HostId); err != nil { - h.onFail(fmt.Sprintf("Sync device %s: %v", dev.String(), err)) - return + // sync each isolated device found + updateDevs := jsonutils.NewArray() + for _, dev := range h.IsolatedDeviceMan.GetDevices() { + if obj, err := isolated_device.SyncDeviceInfo(h.GetSession(), h.HostId, dev); err != nil { + return nil, errors.Wrapf(err, "Sync device %s", dev) + } else { + updateDevs.Add(obj) } } - - h.deployAdminAuthorizedKeys() + return updateDevs, nil } func (h *SHostInfo) deployAdminAuthorizedKeys() { diff --git a/pkg/hostman/hostutils/hostutils.go b/pkg/hostman/hostutils/hostutils.go index 1c2de8b9b3..42634af2a8 100644 --- a/pkg/hostman/hostutils/hostutils.go +++ b/pkg/hostman/hostutils/hostutils.go @@ -52,7 +52,7 @@ type IHost interface { StartDHCPServer() GetBridgeDev(bridge string) hostbridge.IBridgeDriver - GetIsolatedDeviceManager() *isolated_device.IsolatedDeviceManager + GetIsolatedDeviceManager() isolated_device.IsolatedDeviceManager SyncRootPartitionUsedCapacity() error diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go new file mode 100644 index 0000000000..2ec602eb30 --- /dev/null +++ b/pkg/hostman/isolated_device/gpu.go @@ -0,0 +1,534 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package isolated_device + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/util/sets" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + o "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/procutils" + "yunion.io/x/onecloud/pkg/util/regutils2" +) + +const ( + CLASS_CODE_VGA = "0300" + CLASS_CODE_3D = "0302" +) + +const ( + BUSID_REGEX = `[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]` + CODE_REGEX = `[0-9a-fA-F]{4}` + LABEL_REGEX = `[\w+\ \.\,\:\+\&\-\/\[\]\(\)]+` + + VFIO_PCI_KERNEL_DRIVER = "vfio-pci" + DEFAULT_VGA_CMD = " -vga std" + // 在qemu/kvm下模拟Windows Hyper-V的一些半虚拟化特性，以便更好地使用Win虚拟机 + // http://blog.wikichoon.com/2014/07/enabling-hyper-v-enlightenments-with-kvm.html + // 但实际测试不行，虚拟机不能运行nvidia驱动 + // DEFAULT_CPU_CMD = "host,kvm=off,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time" + DEFAULT_CPU_CMD = "host,kvm=off" +) + +func getPassthroughGPUS() ([]*PCIDevice, error) { + gpus, err := detectGPUS() + if err != nil { + return nil, err + } + ret := []*PCIDevice{} + for _, dev := range gpus { + if drv, err := dev.getKernelDriver(); err != nil { + log.Errorf("Device %#v get kernel driver error: %v", dev, err) + } else if drv == VFIO_PCI_KERNEL_DRIVER { + ret = append(ret, dev) + } else { + log.Warningf("GPU %v use kernel driver %q, skip it", dev, drv) + } + } + return ret, nil +} + +func detectGPUS() ([]*PCIDevice, error) { + lines, err := getGPUPCIStr() + if err != nil { + return nil, err + } + devs := []*PCIDevice{} + for _, line := range lines { + dev, err := NewPCIDevice(line) + if err != nil { + return nil, err + } + devs = append(devs, dev) + } + return devs, nil +} + +func getGPUPCIStr() ([]string, error) { + ret, err := bashOutput("lspci -nnmm | egrep '3D|VGA'") + if err != nil { + return nil, err + } + lines := []string{} + for _, l := range ret { + if len(l) != 0 { + lines = append(lines, l) + } + } + return lines, err +} + +type PCIDevice struct { + Addr string `json:"bus_id"` + ClassName string `json:"class_name"` + ClassCode string `json:"class_code"` + VendorName string `json:"vendor_name"` + VendorId string `json:"vendor_id"` + DeviceName string `json:"device_name"` + DeviceId string `json:"device_id"` + SubvendorName string `json:"subvendor_name"` + SubvendorId string `json:"subvendor_id"` + SubdeviceName string `json:"subdevice_name"` + SubdeviceId string `json:"subdevice_id"` + ModelName string `json:"model_name"` + + RestIOMMUGroupDevs []*PCIDevice `json:"-"` +} + +func NewPCIDevice(line string) (*PCIDevice, error) { + dev := parseLspci(line) + if err := dev.checkSameIOMMUGroupDevice(); err != nil { + return nil, err + } + if err := dev.forceBindVFIOPCIDriver(o.HostOptions.UseBootVga); err != nil { + return nil, fmt.Errorf("Force bind vfio-pci driver: %v", err) + } + return dev, nil +} + +func NewPCIDevice2(line string) *PCIDevice { + return parseLspci(line) +} + +type sGPUBaseDevice struct { + *sBaseDevice +} + +func newGPUBaseDevice(dev *PCIDevice, devType string) *sGPUBaseDevice { + return &sGPUBaseDevice{ + sBaseDevice: newBaseDevice(dev, devType), + } +} + +func (dev *sGPUBaseDevice) GetCPUCmd() string { + return DEFAULT_CPU_CMD +} + +func (dev *sGPUBaseDevice) GetVGACmd() string { + return DEFAULT_VGA_CMD +} + +func (dev *sGPUBaseDevice) DetectByAddr() error { + _, err := detectPCIDevByAddr(dev.GetAddr()) + return err +} + +func (dev *sGPUBaseDevice) CustomProbe() error { + // vfio kernel driver check + for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} { + if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", driver).Run(); err != nil { + return fmt.Errorf("modprobe %s: %v", driver, err) + } + } + // grub check + grubCmdline, err := fileutils2.FileGetContents("/proc/cmdline") + if err != nil { + return err + } + grubCmdline = strings.TrimSpace(grubCmdline) + params := sets.NewString(strings.Split(grubCmdline, " ")...) + if !params.IsSuperset(sets.NewString("intel_iommu=on", + "vfio_iommu_type1.allow_unsafe_interrupts=1")) { + return fmt.Errorf("Some GRUB_CMDLINE iommu parameters are missing") + } + isNouveauBlacklisted := false + if params.IsSuperset(sets.NewString("rdblacklist=nouveau", "nouveau.modeset=0")) || + params.IsSuperset(sets.NewString("rd.driver.blacklist=nouveau", "nouveau.modeset=0")) { + isNouveauBlacklisted = true + } + if !isNouveauBlacklisted { + return fmt.Errorf("Some GRUB_CMDLINE nouveau_blacklisted parameters are missing") + } + driver, err := dev.GetKernelDriver() + if err != nil { + return err + } + if driver != "" && driver != VFIO_PCI_KERNEL_DRIVER { + return fmt.Errorf("GPU is occupied by another driver: %s", driver) + } + if driver == "" { + //fileutils2.FilePutContents( + //fmt.Sprintf("%s\n", strings.Replace(dev.GetVendorDeviceId(), ":", " ", -1)), + //false) + } + return nil +} + +func (dev *sGPUBaseDevice) GetHotPlugOptions() ([]*HotPlugOption, error) { + return nil, fmt.Errorf("Not implemented") +} + +func (dev *sGPUBaseDevice) GetHotUnplugOptions() ([]*HotUnplugOption, error) { + return nil, fmt.Errorf("Not implemented") +} + +type sGPUVGADevice struct { + *sGPUBaseDevice +} + +func (gpu *sGPUVGADevice) GetDeviceType() string { + return api.GPU_VGA_TYPE +} + +func (gpu *sGPUVGADevice) GetVGACmd() string { + return " -vga none" +} + +func getGuestAddr(index int) string { + vAddr := fmt.Sprintf("0x%x", 21+index) // from 0x15 above + return vAddr +} + +func (gpu *sGPUVGADevice) GetPassthroughCmd(index int) string { + // vAddr := getGuestAddr(index) + return fmt.Sprintf(" -device vfio-pci,host=%s,multifunction=on,x-vga=on", gpu.GetAddr()) +} + +func (gpu *sGPUVGADevice) CustomProbe() error { + _, err := bashOutput(`cat /boot/cfg-$(uname -r) | grep -E "^CONFIG_VFIO_PCI_VGA=y"`) + if err != nil { + return fmt.Errorf("CONFIG_VFIO_PCI_VGA=y needs to be set in kernel compiling parameters") + } + return nil +} + +type sGPUHPCDevice struct { + *sGPUBaseDevice +} + +func NewGPUHPCDevice(dev *PCIDevice) *sGPUHPCDevice { + gpuDev := &sGPUHPCDevice{ + sGPUBaseDevice: newGPUBaseDevice(dev, api.GPU_HPC_TYPE), + } + return gpuDev +} + +func (gpu *sGPUHPCDevice) GetPassthroughCmd(index int) string { + // vAddr := getGuestAddr(index) + return fmt.Sprintf(" -device vfio-pci,host=%s,multifunction=on", gpu.GetAddr()) +} + +func gpuPCIString() ([]string, error) { + lines, err := bashOutput("lspci -nnmm | egrep '3D|VGA'") + if err != nil { + return nil, fmt.Errorf("Get GPU PCI: %v", err) + } + ret := []string{} + for _, line := range lines { + if len(line) != 0 { + ret = append(ret, line) + } + } + return ret, nil +} + +func gpuPCIAddr() ([]string, error) { + lines, err := gpuPCIString() + if err != nil { + return nil, err + } + addrs := []string{} + for _, line := range lines { + addrs = append(addrs, strings.Split(line, " ")[0]) + } + return addrs, nil +} + +// parseLspci parse one line output of `lspci -nnmm` +func parseLspci(line string) *PCIDevice { + itemRegex := `(?P(` + BUSID_REGEX + `))` + + `\ "(?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\]"` + + `\ "(?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\]"` + + `\ "(?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\]"` + + `\ .*\"((?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\])*"` + + `\ "((?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\])*` + ret := regutils2.SubGroupMatch(itemRegex, line) + dev := PCIDevice{} + jsonutils.Marshal(ret).Unmarshal(&dev) + deviceRegex := `(?P` + LABEL_REGEX + `)\ \[(?P` + LABEL_REGEX + `)\]` + if ret := regutils2.SubGroupMatch(deviceRegex, dev.DeviceName); len(ret) != 0 { + dev.ModelName = ret["model_name"] + } + return &dev +} + +func (d *PCIDevice) GetVendorDeviceId() string { + return fmt.Sprintf("%s:%s", d.VendorId, d.DeviceId) +} + +// checkSameIOMMUGroupDevice check related device like Audio in same iommu group +// e.g. +// 41:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP107 [GeForce GTX 1050 Ti] [10de:1c82] (rev a1) +// 41:00.1 Audio device [0403]: NVIDIA Corporation GP107GL High Definition Audio Controller [10de:0fb9] (rev a1) +func (d *PCIDevice) checkSameIOMMUGroupDevice() error { + group, err := NewIOMMUGroup() + if err != nil { + return fmt.Errorf("IOMMUGroup FindSameGroupDevs: %v", err) + } + d.RestIOMMUGroupDevs = group.FindSameGroupDevs(d.Addr, d.VendorId) + return nil +} + +func (d *PCIDevice) IsBootVGA() (bool, error) { + addr := d.Addr + output, err := procutils.NewCommand("find", "/sys/devices", "-name", "boot_vga").Output() + if err != nil { + if exiterr, ok := err.(*exec.ExitError); ok { + if code, ok := exiterr.Sys().(syscall.WaitStatus); ok && + code.ExitStatus() == 1 && strings.Contains(string(output), "No such file or directory") { + log.Warningf("find boot vga %s", output) + } else { + return false, err + } + } else { + return false, err + } + } + paths := ParseOutput(output) + for _, p := range paths { + if strings.Contains(p, addr) && !strings.Contains(p, "No such file or directory") { + if content, err := fileutils2.FileGetContents(p); err != nil { + return false, err + } else { + if len(content) > 0 && strings.HasPrefix(content, "1") { + log.Infof("PCI address %s is boot_vga: %s", addr, p) + return true, nil + } + } + } + } + return false, nil +} + +func (d *PCIDevice) forceBindVFIOPCIDriver(useBootVGA bool) error { + if !utils.IsInStringArray(d.ClassCode, []string{CLASS_CODE_VGA, CLASS_CODE_VGA}) { + return nil + } + isBootVGA, err := d.IsBootVGA() + if err != nil { + return err + } + if !useBootVGA && isBootVGA { + log.Infof("%#v is boot vga card, skip it", d) + return nil + } + if d.IsVFIOPCIDriverUsed() { + log.Infof("%s already use vfio-pci driver", d) + return nil + } + + devs := []*PCIDevice{} + devs = append(devs, d.RestIOMMUGroupDevs...) + devs = append(devs, d) + for _, dev := range devs { + if err := dev.bindAddrVFIOPCI(); err != nil { + return fmt.Errorf("bind %s vfio-pci driver: %v", dev, err) + } + } + return nil +} + +func (d *PCIDevice) bindAddrVFIOPCI() error { + if err := d.unbindDriver(); err != nil { + return fmt.Errorf("unbindDriver: %v", err) + } + if err := d.bindDriver(); err != nil { + return fmt.Errorf("bindDriver: %v", err) + } + return nil +} + +func (d *PCIDevice) unbindDriver() error { + driver, err := d.getKernelDriver() + if err != nil { + return err + } + if len(driver) != 0 { + if err := fileutils2.FilePutContents( + fmt.Sprintf("/sys/bus/pci/devices/0000:%s/driver/unbind", d.Addr), + fmt.Sprintf("0000:%s", d.Addr), false); err != nil { + return fmt.Errorf("unbindDriver: %v", err) + } + } + return nil +} + +func (d *PCIDevice) bindDriver() error { + vendorDevId := fmt.Sprintf("%s %s", d.VendorId, d.DeviceId) + return fileutils2.FilePutContents( + "/sys/bus/pci/drivers/vfio-pci/new_id", + fmt.Sprintf("%s\n", vendorDevId), + false, + ) +} + +func (d *PCIDevice) String() string { + return jsonutils.Marshal(d).String() +} + +func (d *PCIDevice) IsVFIOPCIDriverUsed() bool { + driver, _ := d.getKernelDriver() + if driver != VFIO_PCI_KERNEL_DRIVER { + return false + } + for _, dev := range d.RestIOMMUGroupDevs { + driver, _ := dev.getKernelDriver() + if driver != VFIO_PCI_KERNEL_DRIVER { + return false + } + } + return true +} + +func (d *PCIDevice) getKernelDriver() (string, error) { + prompt := "Kernel driver in use: " + lines, err := bashOutput(fmt.Sprintf("lspci -k -s %s", d.Addr)) + if err != nil { + return "", err + } + for _, line := range lines { + begin := strings.Index(line, prompt) + if begin >= 0 { + end := begin + len(prompt) + return line[end:], nil + } + } + // no driver in use + return "", nil +} + +type IOMMUGroup struct { + // busId: group + group map[string]string +} + +func NewIOMMUGroup() (*IOMMUGroup, error) { + devPaths := "/sys/kernel/iommu_groups/" + dict := make(map[string]string) + err := filepath.Walk(devPaths, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + parts := strings.Split(path, "/") + group := parts[4] + busId := parts[len(parts)-1] + dict[busId] = group + return nil + }) + if err != nil { + return nil, err + } + return &IOMMUGroup{group: dict}, nil +} + +func (g *IOMMUGroup) ListDevices(groupNum, selfAddr, vendorId string) []*PCIDevice { + ret := []string{} + for busId, group := range g.group { + if groupNum == group { + ret = append(ret, busId) + } + } + + devs := []*PCIDevice{} + for _, addr := range ret { + if addr == selfAddr { + continue + } + dev, _ := detectPCIDevByAddrWithoutIOMMUGroup(addr[5:]) + if dev != nil { + if dev.VendorId == vendorId { + devs = append(devs, dev) + } else { + log.Warningf("Skip append %q iommu_group[%s] device %s", selfAddr, groupNum, dev.String()) + } + } + } + return devs +} + +func (g *IOMMUGroup) FindSameGroupDevs(devAddr string, vendorId string) []*PCIDevice { + // devAddr: '0000:3f:0f.3' or '3f:0f.3' format + if len(devAddr) == 7 { + devAddr = fmt.Sprintf("0000:%s", devAddr) + } + group, ok := g.group[devAddr] + if !ok { + return nil + } + return g.ListDevices(group, devAddr, vendorId) +} + +func (g *IOMMUGroup) String() string { + return jsonutils.Marshal(g.group).PrettyString() +} + +func detectPCIDevByAddr(addr string) (*PCIDevice, error) { + ret, err := bashOutput(fmt.Sprintf("lspci -nnmm -s %s", addr)) + if err != nil { + return nil, err + } + return NewPCIDevice(strings.Join(ret, "")) +} + +func detectPCIDevByAddrWithoutIOMMUGroup(addr string) (*PCIDevice, error) { + ret, err := bashOutput(fmt.Sprintf("lspci -nnmm -s %s", addr)) + if err != nil { + return nil, err + } + return NewPCIDevice2(strings.Join(ret, "")), nil +} + +func getDeviceCmd(dev IDevice, index int) string { + passthroughCmd := dev.GetPassthroughCmd(index) + groupDevCmd := dev.GetIOMMUGroupDeviceCmd() + if len(groupDevCmd) != 0 { + passthroughCmd = fmt.Sprintf("%s%s", passthroughCmd, groupDevCmd) + } + return passthroughCmd +} diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 92c407bbbd..17d6ec71bc 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -16,45 +16,19 @@ package isolated_device import ( "fmt" - "os" - "os/exec" - "path/filepath" "strings" - "syscall" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/util/sets" - "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" - o "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/mcclient" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" - "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" - "yunion.io/x/onecloud/pkg/util/regutils2" ) const ( - CLASS_CODE_VGA = "0300" - CLASS_CODE_3D = "0302" -) - -const ( - BUSID_REGEX = `[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]` - CODE_REGEX = `[0-9a-fA-F]{4}` - LABEL_REGEX = `[\w+\ \.\,\:\+\&\-\/\[\]\(\)]+` - - VFIO_PCI_KERNEL_DRIVER = "vfio-pci" - DEFAULT_VGA_CMD = " -vga std" - // 在qemu/kvm下模拟Windows Hyper-V的一些半虚拟化特性，以便更好地使用Win虚拟机 - // http://blog.wikichoon.com/2014/07/enabling-hyper-v-enlightenments-with-kvm.html - // 但实际测试不行，虚拟机不能运行nvidia驱动 - // DEFAULT_CPU_CMD = "host,kvm=off,hv_relaxed,hv_spinlocks=0x1fff,hv_vapic,hv_time" - DEFAULT_CPU_CMD = "host,kvm=off" - RESOURCE = "isolated_devices" ) @@ -73,65 +47,109 @@ type IHost interface { GetSession() *mcclient.ClientSession } +type HotPlugOption struct { + Device string + Options map[string]interface{} +} + +type HotUnplugOption struct { + Id string +} + type IDevice interface { String() string GetCloudId() string + GetHostId() string + SetHostId(hId string) + GetGuestId() string GetVendorDeviceId() string GetAddr() string GetDeviceType() string + GetModelName() string CustomProbe() error SetDeviceInfo(info CloudDeviceInfo) SetDetectedOnHost(isDetected bool) + DetectByAddr() error GetPassthroughCmd(index int) string GetIOMMUGroupDeviceCmd() string GetVGACmd() string GetCPUCmd() string - SyncDeviceInfo(session *mcclient.ClientSession, hostId string) error + + GetHotPlugOptions() ([]*HotPlugOption, error) + GetHotUnplugOptions() ([]*HotUnplugOption, error) } -type IsolatedDeviceManager struct { +type IsolatedDeviceManager interface { + GetDevices() []IDevice + GetDeviceByIdent(vendorDevId string, addr string) IDevice + ProbePCIDevices(skipGPUs, skipUSBs bool) error + StartDetachTask() + BatchCustomProbe() error + AppendDetachedDevice(dev *CloudDeviceInfo) + GetQemuParams(devAddrs []string) *QemuParams +} + +type isolatedDeviceManager struct { host IHost - Devices []IDevice + devices []IDevice DetachedDevices []*CloudDeviceInfo } -func NewManager(host IHost) *IsolatedDeviceManager { - man := &IsolatedDeviceManager{ +func NewManager(host IHost) IsolatedDeviceManager { + man := &isolatedDeviceManager{ host: host, - Devices: make([]IDevice, 0), + devices: make([]IDevice, 0), DetachedDevices: make([]*CloudDeviceInfo, 0), } // Do probe laster - Qiu Jian - // err := man.fillPCIDevices() return man } -func (man *IsolatedDeviceManager) ProbePCIDevices() error { - if len(man.Devices) > 0 { +func (man *isolatedDeviceManager) GetDevices() []IDevice { + return man.devices +} + +func (man *isolatedDeviceManager) ProbePCIDevices(skipGPUs, skipUSBs bool) error { + if len(man.devices) > 0 { // already probed, skip return nil } - // only support gpu by now - gpus, err := getPassthroughGPUS() - if err != nil { - // ignore getPassthroughGPUS error on old machines without VGA devices - log.Errorf("getPassthroughGPUS: %v", err) - return nil + + if !skipGPUs { + gpus, err := getPassthroughGPUS() + if err != nil { + // ignore getPassthroughGPUS error on old machines without VGA devices + log.Errorf("getPassthroughGPUS: %v", err) + return nil + } + for idx, gpu := range gpus { + man.devices = append(man.devices, NewGPUHPCDevice(gpu)) + log.Infof("Add GPU device: %d => %#v", idx, gpu) + } } - for idx, gpu := range gpus { - man.Devices = append(man.Devices, NewGPUHPCDevice(gpu)) - log.Infof("Add GPU device: %d => %#v", idx, gpu) + + if !skipUSBs { + usbs, err := getPassthroughUSBs() + if err != nil { + log.Errorf("getPassthroughUSBs: %v", err) + return nil + } + for idx, usb := range usbs { + man.devices = append(man.devices, usb) + log.Infof("Add USB device: %d => %#v", idx, usb) + } } + return nil } -func (man *IsolatedDeviceManager) getSession() *mcclient.ClientSession { +func (man *isolatedDeviceManager) getSession() *mcclient.ClientSession { return man.host.GetSession() } -func (man *IsolatedDeviceManager) GetDeviceByIdent(vendorDevId string, addr string) IDevice { - for _, dev := range man.Devices { +func (man *isolatedDeviceManager) GetDeviceByIdent(vendorDevId string, addr string) IDevice { + for _, dev := range man.devices { if dev.GetVendorDeviceId() == vendorDevId && dev.GetAddr() == addr { return dev } @@ -139,8 +157,8 @@ func (man *IsolatedDeviceManager) GetDeviceByIdent(vendorDevId string, addr stri return nil } -func (man *IsolatedDeviceManager) GetDeviceByVendorDevId(vendorDevId string) IDevice { - for _, dev := range man.Devices { +func (man *isolatedDeviceManager) GetDeviceByVendorDevId(vendorDevId string) IDevice { + for _, dev := range man.devices { if dev.GetVendorDeviceId() == vendorDevId { return dev } @@ -148,8 +166,8 @@ func (man *IsolatedDeviceManager) GetDeviceByVendorDevId(vendorDevId string) IDe return nil } -func (man *IsolatedDeviceManager) GetDeviceByAddr(addr string) IDevice { - for _, dev := range man.Devices { +func (man *isolatedDeviceManager) GetDeviceByAddr(addr string) IDevice { + for _, dev := range man.devices { if dev.GetAddr() == addr { return dev } @@ -157,8 +175,8 @@ func (man *IsolatedDeviceManager) GetDeviceByAddr(addr string) IDevice { return nil } -func (man *IsolatedDeviceManager) BatchCustomProbe() error { - for _, dev := range man.Devices { +func (man *isolatedDeviceManager) BatchCustomProbe() error { + for _, dev := range man.devices { if err := dev.CustomProbe(); err != nil { return err } @@ -166,12 +184,12 @@ func (man *IsolatedDeviceManager) BatchCustomProbe() error { return nil } -func (man *IsolatedDeviceManager) AppendDetachedDevice(dev *CloudDeviceInfo) { +func (man *isolatedDeviceManager) AppendDetachedDevice(dev *CloudDeviceInfo) { dev.DetectedOnHost = false man.DetachedDevices = append(man.DetachedDevices, dev) } -func (man *IsolatedDeviceManager) StartDetachTask() { +func (man *isolatedDeviceManager) StartDetachTask() { if len(man.DetachedDevices) == 0 { return } @@ -190,7 +208,7 @@ func (man *IsolatedDeviceManager) StartDetachTask() { }() } -func (man *IsolatedDeviceManager) GetQemuParams(devAddrs []string) *QemuParams { +func (man *isolatedDeviceManager) GetQemuParams(devAddrs []string) *QemuParams { return getQemuParams(man, devAddrs) } @@ -203,8 +221,19 @@ type sBaseDevice struct { detectedOnHost bool } -func newBaseDevice(dev *PCIDevice) *sBaseDevice { - return &sBaseDevice{dev: dev} +func newBaseDevice(dev *PCIDevice, devType string) *sBaseDevice { + return &sBaseDevice{ + dev: dev, + devType: devType, + } +} + +func (dev *sBaseDevice) GetHostId() string { + return dev.hostId +} + +func (dev *sBaseDevice) SetHostId(hId string) { + dev.hostId = hId } func (dev *sBaseDevice) String() string { @@ -230,19 +259,17 @@ func (dev *sBaseDevice) SetDetectedOnHost(probe bool) { dev.detectedOnHost = probe } -func (dev *sBaseDevice) SyncDeviceInfo(session *mcclient.ClientSession, hostId string) error { - if len(dev.hostId) == 0 { - dev.hostId = hostId +func SyncDeviceInfo(session *mcclient.ClientSession, hostId string, dev IDevice) (jsonutils.JSONObject, error) { + if len(dev.GetHostId()) == 0 { + dev.SetHostId(hostId) } - data := dev.GetApiResourceData() + data := GetApiResourceData(dev) if len(dev.GetCloudId()) != 0 { log.Infof("Update %s isolated_device: %s", dev.GetCloudId(), data.String()) - _, err := modules.IsolatedDevices.Update(session, dev.GetCloudId(), data) - return err + return modules.IsolatedDevices.Update(session, dev.GetCloudId(), data) } log.Infof("Create new isolated_device: %s", data.String()) - _, err := modules.IsolatedDevices.Create(session, data) - return err + return modules.IsolatedDevices.Create(session, data) } func (dev *sBaseDevice) GetCloudId() string { @@ -261,39 +288,42 @@ func (dev *sBaseDevice) GetDeviceType() string { return dev.devType } -func (dev *sBaseDevice) GetApiResourceData() jsonutils.JSONObject { +func (dev *sBaseDevice) GetModelName() string { + return dev.dev.ModelName +} + +func (dev *sBaseDevice) GetGuestId() string { + return dev.guestId +} + +func GetApiResourceData(dev IDevice) *jsonutils.JSONDict { data := map[string]interface{}{ "dev_type": dev.GetDeviceType(), "addr": dev.GetAddr(), - "model": dev.dev.ModelName, + "model": dev.GetModelName(), "vendor_device_id": dev.GetVendorDeviceId(), } detected := false - if _, err := detectPCIDevByAddr(dev.GetAddr()); err == nil { + if err := dev.DetectByAddr(); err == nil { detected = true } data["detected_on_host"] = detected - if len(dev.cloudId) != 0 { - data["id"] = dev.cloudId + if len(dev.GetCloudId()) != 0 { + data["id"] = dev.GetCloudId() } - if len(dev.hostId) != 0 { - data["host_id"] = dev.hostId + if len(dev.GetHostId()) != 0 { + data["host_id"] = dev.GetHostId() } - if len(dev.guestId) != 0 { - data["guest_id"] = dev.guestId + if len(dev.GetGuestId()) != 0 { + data["guest_id"] = dev.GetGuestId() } - return jsonutils.Marshal(data) + return jsonutils.Marshal(data).(*jsonutils.JSONDict) } func (dev *sBaseDevice) GetKernelDriver() (string, error) { return dev.dev.getKernelDriver() } -func (dev *sBaseDevice) IsPassthroughAble() bool { - driver, _ := dev.GetKernelDriver() - return driver == VFIO_PCI_KERNEL_DRIVER -} - func (dev *sBaseDevice) getVFIODeviceCmd(addr string) string { return fmt.Sprintf(" -device vfio-pci,host=%s", addr) } @@ -319,116 +349,10 @@ func (dev *sBaseDevice) GetIOMMUGroupDeviceCmd() string { return strings.Join(cmds, "") } -type sGPUBaseDevice struct { - *sBaseDevice -} - -func newGPUBaseDevice(dev *PCIDevice) *sGPUBaseDevice { - return &sGPUBaseDevice{ - sBaseDevice: newBaseDevice(dev), - } -} - -func (dev *sGPUBaseDevice) GetCPUCmd() string { - return DEFAULT_CPU_CMD -} - -func (dev *sGPUBaseDevice) GetVGACmd() string { - return DEFAULT_VGA_CMD -} - -func (dev *sGPUBaseDevice) CustomProbe() error { - // vfio kernel driver check - for _, driver := range []string{"vfio", "vfio_iommu_type1", "vfio-pci"} { - if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", driver).Run(); err != nil { - return fmt.Errorf("modprobe %s: %v", driver, err) - } - } - // grub check - grubCmdline, err := fileutils2.FileGetContents("/proc/cmdline") - if err != nil { - return err - } - grubCmdline = strings.TrimSpace(grubCmdline) - params := sets.NewString(strings.Split(grubCmdline, " ")...) - if !params.IsSuperset(sets.NewString("intel_iommu=on", - "vfio_iommu_type1.allow_unsafe_interrupts=1")) { - return fmt.Errorf("Some GRUB_CMDLINE iommu parameters are missing") - } - isNouveauBlacklisted := false - if params.IsSuperset(sets.NewString("rdblacklist=nouveau", "nouveau.modeset=0")) || - params.IsSuperset(sets.NewString("rd.driver.blacklist=nouveau", "nouveau.modeset=0")) { - isNouveauBlacklisted = true - } - if !isNouveauBlacklisted { - return fmt.Errorf("Some GRUB_CMDLINE nouveau_blacklisted parameters are missing") - } - driver, err := dev.GetKernelDriver() - if err != nil { - return err - } - if driver != "" && driver != VFIO_PCI_KERNEL_DRIVER { - return fmt.Errorf("GPU is occupied by another driver: %s", driver) - } - if driver == "" { - //fileutils2.FilePutContents( - //fmt.Sprintf("%s\n", strings.Replace(dev.GetVendorDeviceId(), ":", " ", -1)), - //false) - } +func (dev *sBaseDevice) DetectByAddr() error { return nil } -type sGPUVGADevice struct { - *sGPUBaseDevice -} - -func (gpu *sGPUVGADevice) GetDeviceType() string { - return api.GPU_VGA_TYPE -} - -func (gpu *sGPUVGADevice) GetVGACmd() string { - return " -vga none" -} - -func getGuestAddr(index int) string { - vAddr := fmt.Sprintf("0x%x", 21+index) // from 0x15 above - return vAddr -} - -func (gpu *sGPUVGADevice) GetPassthroughCmd(index int) string { - // vAddr := getGuestAddr(index) - return fmt.Sprintf(" -device vfio-pci,host=%s,multifunction=on,x-vga=on", gpu.GetAddr()) -} - -func (gpu *sGPUVGADevice) CustomProbe() error { - _, err := bashOutput(`cat /boot/cfg-$(uname -r) | grep -E "^CONFIG_VFIO_PCI_VGA=y"`) - if err != nil { - return fmt.Errorf("CONFIG_VFIO_PCI_VGA=y needs to be set in kernel compiling parameters") - } - return nil -} - -type sGPUHPCDevice struct { - *sGPUBaseDevice -} - -func NewGPUHPCDevice(dev *PCIDevice) *sGPUHPCDevice { - gpuDev := &sGPUHPCDevice{ - sGPUBaseDevice: newGPUBaseDevice(dev), - } - gpuDev.devType = gpuDev.GetDeviceType() - return gpuDev -} - -func (gpu *sGPUHPCDevice) GetDeviceType() string { - return api.GPU_HPC_TYPE -} - -func (gpu *sGPUHPCDevice) GetPassthroughCmd(index int) string { - // vAddr := getGuestAddr(index) - return fmt.Sprintf(" -device vfio-pci,host=%s,multifunction=on", gpu.GetAddr()) -} - func ParseOutput(output []byte) []string { lines := make([]string, 0) for _, line := range strings.Split(string(output), "\n") { @@ -447,395 +371,49 @@ func bashOutput(cmd string) ([]string, error) { } } -func gpuPCIString() ([]string, error) { - lines, err := bashOutput("lspci -nnmm | egrep '3D|VGA'") - if err != nil { - return nil, fmt.Errorf("Get GPU PCI: %v", err) - } - ret := []string{} - for _, line := range lines { - if len(line) != 0 { - ret = append(ret, line) - } - } - return ret, nil -} - -func gpuPCIAddr() ([]string, error) { - lines, err := gpuPCIString() - if err != nil { - return nil, err - } - addrs := []string{} - for _, line := range lines { - addrs = append(addrs, strings.Split(line, " ")[0]) - } - return addrs, nil -} - -type PCIDevice struct { - Addr string `json:"bus_id"` - ClassName string `json:"class_name"` - ClassCode string `json:"class_code"` - VendorName string `json:"vendor_name"` - VendorId string `json:"vendor_id"` - DeviceName string `json:"device_name"` - DeviceId string `json:"device_id"` - SubvendorName string `json:"subvendor_name"` - SubvendorId string `json:"subvendor_id"` - SubdeviceName string `json:"subdevice_name"` - SubdeviceId string `json:"subdevice_id"` - ModelName string `json:"model_name"` - - RestIOMMUGroupDevs []*PCIDevice `json:"-"` -} - -func NewPCIDevice(line string) (*PCIDevice, error) { - dev := parseLspci(line) - if err := dev.checkSameIOMMUGroupDevice(); err != nil { - return nil, err - } - if err := dev.forceBindVFIOPCIDriver(o.HostOptions.UseBootVga); err != nil { - return nil, fmt.Errorf("Force bind vfio-pci driver: %v", err) - } - return dev, nil -} - -func NewPCIDevice2(line string) *PCIDevice { - return parseLspci(line) -} - -// parseLspci parse one line output of `lspci -nnmm` -func parseLspci(line string) *PCIDevice { - itemRegex := `(?P(` + BUSID_REGEX + `))` + - `\ "(?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\]"` + - `\ "(?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\]"` + - `\ "(?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\]"` + - `\ .*\"((?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\])*"` + - `\ "((?P` + LABEL_REGEX + `)\ \[(?P` + CODE_REGEX + `)\])*` - ret := regutils2.SubGroupMatch(itemRegex, line) - dev := PCIDevice{} - jsonutils.Marshal(ret).Unmarshal(&dev) - deviceRegex := `(?P` + LABEL_REGEX + `)\ \[(?P` + LABEL_REGEX + `)\]` - if ret := regutils2.SubGroupMatch(deviceRegex, dev.DeviceName); len(ret) != 0 { - dev.ModelName = ret["model_name"] - } - return &dev -} - -func (d *PCIDevice) GetVendorDeviceId() string { - return fmt.Sprintf("%s:%s", d.VendorId, d.DeviceId) -} - -// checkSameIOMMUGroupDevice check related device like Audio in same iommu group -// e.g. -// 41:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP107 [GeForce GTX 1050 Ti] [10de:1c82] (rev a1) -// 41:00.1 Audio device [0403]: NVIDIA Corporation GP107GL High Definition Audio Controller [10de:0fb9] (rev a1) -func (d *PCIDevice) checkSameIOMMUGroupDevice() error { - group, err := NewIOMMUGroup() - if err != nil { - return fmt.Errorf("IOMMUGroup FindSameGroupDevs: %v", err) - } - d.RestIOMMUGroupDevs = group.FindSameGroupDevs(d.Addr, d.VendorId) - return nil -} - -func (d *PCIDevice) IsBootVGA() (bool, error) { - addr := d.Addr - output, err := procutils.NewCommand("find", "/sys/devices", "-name", "boot_vga").Output() - if err != nil { - if exiterr, ok := err.(*exec.ExitError); ok { - if code, ok := exiterr.Sys().(syscall.WaitStatus); ok && - code.ExitStatus() == 1 && strings.Contains(string(output), "No such file or directory") { - log.Warningf("find boot vga %s", output) - } else { - return false, err - } - } else { - return false, err - } - } - paths := ParseOutput(output) - for _, p := range paths { - if strings.Contains(p, addr) && !strings.Contains(p, "No such file or directory") { - if content, err := fileutils2.FileGetContents(p); err != nil { - return false, err - } else { - if len(content) > 0 && strings.HasPrefix(content, "1") { - log.Infof("PCI address %s is boot_vga: %s", addr, p) - return true, nil - } - } - } - } - return false, nil -} - -func (d *PCIDevice) forceBindVFIOPCIDriver(useBootVGA bool) error { - if !utils.IsInStringArray(d.ClassCode, []string{CLASS_CODE_VGA, CLASS_CODE_VGA}) { - return nil - } - isBootVGA, err := d.IsBootVGA() - if err != nil { - return err - } - if !useBootVGA && isBootVGA { - log.Infof("%#v is boot vga card, skip it", d) - return nil - } - if d.IsVFIOPCIDriverUsed() { - log.Infof("%s already use vfio-pci driver", d) - return nil - } - - devs := []*PCIDevice{} - devs = append(devs, d.RestIOMMUGroupDevs...) - devs = append(devs, d) - for _, dev := range devs { - if err := dev.bindAddrVFIOPCI(); err != nil { - return fmt.Errorf("bind %s vfio-pci driver: %v", dev, err) - } - } - return nil -} - -func (d *PCIDevice) bindAddrVFIOPCI() error { - if err := d.unbindDriver(); err != nil { - return fmt.Errorf("unbindDriver: %v", err) - } - if err := d.bindDriver(); err != nil { - return fmt.Errorf("bindDriver: %v", err) - } - return nil -} - -func (d *PCIDevice) unbindDriver() error { - driver, err := d.getKernelDriver() - if err != nil { - return err - } - if len(driver) != 0 { - if err := fileutils2.FilePutContents( - fmt.Sprintf("/sys/bus/pci/devices/0000:%s/driver/unbind", d.Addr), - fmt.Sprintf("0000:%s", d.Addr), false); err != nil { - return fmt.Errorf("unbindDriver: %v", err) - } - } - return nil -} - -func (d *PCIDevice) bindDriver() error { - vendorDevId := fmt.Sprintf("%s %s", d.VendorId, d.DeviceId) - return fileutils2.FilePutContents( - "/sys/bus/pci/drivers/vfio-pci/new_id", - fmt.Sprintf("%s\n", vendorDevId), - false, - ) -} - -func (d *PCIDevice) String() string { - return jsonutils.Marshal(d).String() -} - -func (d *PCIDevice) IsVFIOPCIDriverUsed() bool { - driver, _ := d.getKernelDriver() - if driver != VFIO_PCI_KERNEL_DRIVER { - return false - } - for _, dev := range d.RestIOMMUGroupDevs { - driver, _ := dev.getKernelDriver() - if driver != VFIO_PCI_KERNEL_DRIVER { - return false - } - } - return true -} - -func (d *PCIDevice) getKernelDriver() (string, error) { - prompt := "Kernel driver in use: " - lines, err := bashOutput(fmt.Sprintf("lspci -k -s %s", d.Addr)) - if err != nil { - return "", err - } - for _, line := range lines { - begin := strings.Index(line, prompt) - if begin >= 0 { - end := begin + len(prompt) - return line[end:], nil - } - } - // no driver in use - return "", nil -} - -type IOMMUGroup struct { - // busId: group - group map[string]string -} - -func NewIOMMUGroup() (*IOMMUGroup, error) { - devPaths := "/sys/kernel/iommu_groups/" - dict := make(map[string]string) - err := filepath.Walk(devPaths, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - parts := strings.Split(path, "/") - group := parts[4] - busId := parts[len(parts)-1] - dict[busId] = group - return nil - }) - if err != nil { - return nil, err - } - return &IOMMUGroup{group: dict}, nil -} - -func (g *IOMMUGroup) ListDevices(groupNum, selfAddr, vendorId string) []*PCIDevice { - ret := []string{} - for busId, group := range g.group { - if groupNum == group { - ret = append(ret, busId) - } - } - - devs := []*PCIDevice{} - for _, addr := range ret { - if addr == selfAddr { - continue - } - dev, _ := detectPCIDevByAddrWithoutIOMMUGroup(addr[5:]) - if dev != nil { - if dev.VendorId == vendorId { - devs = append(devs, dev) - } else { - log.Warningf("Skip append %q iommu_group[%s] device %s", selfAddr, groupNum, dev.String()) - } - } - } - return devs -} - -func (g *IOMMUGroup) FindSameGroupDevs(devAddr string, vendorId string) []*PCIDevice { - // devAddr: '0000:3f:0f.3' or '3f:0f.3' format - if len(devAddr) == 7 { - devAddr = fmt.Sprintf("0000:%s", devAddr) - } - group, ok := g.group[devAddr] - if !ok { - return nil - } - return g.ListDevices(group, devAddr, vendorId) -} - -func (g *IOMMUGroup) String() string { - return jsonutils.Marshal(g.group).PrettyString() -} - -func getGPUPCIStr() ([]string, error) { - ret, err := bashOutput("lspci -nnmm | egrep '3D|VGA'") - if err != nil { - return nil, err - } - lines := []string{} - for _, l := range ret { - if len(l) != 0 { - lines = append(lines, l) - } - } - return lines, err -} - -func detectPCIDevByAddr(addr string) (*PCIDevice, error) { - ret, err := bashOutput(fmt.Sprintf("lspci -nnmm -s %s", addr)) - if err != nil { - return nil, err - } - return NewPCIDevice(strings.Join(ret, "")) -} - -func detectPCIDevByAddrWithoutIOMMUGroup(addr string) (*PCIDevice, error) { - ret, err := bashOutput(fmt.Sprintf("lspci -nnmm -s %s", addr)) - if err != nil { - return nil, err - } - return NewPCIDevice2(strings.Join(ret, "")), nil -} - -func detectGPUS() ([]*PCIDevice, error) { - lines, err := getGPUPCIStr() - if err != nil { - return nil, err - } - devs := []*PCIDevice{} - for _, line := range lines { - dev, err := NewPCIDevice(line) - if err != nil { - return nil, err - } - devs = append(devs, dev) - } - return devs, nil -} - -func getPassthroughGPUS() ([]*PCIDevice, error) { - gpus, err := detectGPUS() - if err != nil { - return nil, err - } - ret := []*PCIDevice{} - for _, dev := range gpus { - if drv, err := dev.getKernelDriver(); err != nil { - log.Errorf("Device %#v get kernel driver error: %v", dev, err) - } else if drv == VFIO_PCI_KERNEL_DRIVER { - ret = append(ret, dev) - } else { - log.Warningf("GPU %v use kernel driver %q, skip it", dev, drv) - } - } - return ret, nil -} - type QemuParams struct { Cpu string Vga string Devices []string } -func GetDeviceCmd(dev IDevice, index int) string { - passthroughCmd := dev.GetPassthroughCmd(index) - groupDevCmd := dev.GetIOMMUGroupDeviceCmd() - if len(groupDevCmd) != 0 { - passthroughCmd = fmt.Sprintf("%s%s", passthroughCmd, groupDevCmd) - } - return passthroughCmd -} - -func getQemuParams(man *IsolatedDeviceManager, devAddrs []string) *QemuParams { +func getQemuParams(man *isolatedDeviceManager, devAddrs []string) *QemuParams { if len(devAddrs) == 0 { return nil } devCmds := []string{} cpuCmd := DEFAULT_CPU_CMD vgaCmd := DEFAULT_VGA_CMD - for idx, addr := range devAddrs { + // group by device type firstly + devices := make(map[string][]IDevice, 0) + for _, addr := range devAddrs { dev := man.GetDeviceByAddr(addr) if dev == nil { log.Warningf("IsolatedDeviceManager not found dev %#v, ignore it!", addr) continue } - devCmds = append(devCmds, GetDeviceCmd(dev, idx)) - if dev.GetVGACmd() != vgaCmd && dev.GetDeviceType() == api.GPU_VGA_TYPE { - vgaCmd = dev.GetVGACmd() - } - if dev.GetCPUCmd() != cpuCmd { - cpuCmd = dev.GetCPUCmd() + devType := dev.GetDeviceType() + devs, ok := devices[devType] + if !ok { + devices[devType] = []IDevice{dev} + } else { + devs = append(devs, dev) } } + + for devType, devs := range devices { + log.Debugf("get devices %s command", devType) + for idx, dev := range devs { + devCmds = append(devCmds, getDeviceCmd(dev, idx)) + if dev.GetVGACmd() != vgaCmd && dev.GetDeviceType() == api.GPU_VGA_TYPE { + vgaCmd = dev.GetVGACmd() + } + if dev.GetCPUCmd() != cpuCmd { + cpuCmd = dev.GetCPUCmd() + } + } + } + return &QemuParams{ Cpu: cpuCmd, Vga: vgaCmd, diff --git a/pkg/hostman/isolated_device/usb.go b/pkg/hostman/isolated_device/usb.go new file mode 100644 index 0000000000..bfc4ed29f9 --- /dev/null +++ b/pkg/hostman/isolated_device/usb.go @@ -0,0 +1,212 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package isolated_device + +import ( + "fmt" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/util/regutils2" +) + +type sUSBDevice struct { + *sBaseDevice +} + +// TODO: rename PCIDevice +func newUSBDevice(dev *PCIDevice) *sUSBDevice { + return &sUSBDevice{ + sBaseDevice: newBaseDevice(dev, api.USB_TYPE), + } +} + +func (dev *sUSBDevice) GetCPUCmd() string { + return "" +} + +func (dev *sUSBDevice) GetVGACmd() string { + return "" +} + +func (dev *sUSBDevice) CustomProbe() error { + // do nothing + return nil +} + +func GetUSBDevId(vendorId, devId, bus, addr string) string { + return fmt.Sprintf("dev_%s_%s-%s_%s", vendorId, devId, bus, addr) +} + +func getUSBDevQemuOptions(vendorId, deviceId string, bus, addr string) map[string]interface{} { + id := GetUSBDevId(vendorId, deviceId, bus, addr) + return map[string]interface{}{ + "id": id, + "bus": "usb.0", + "vendorid": fmt.Sprintf("0x%s", vendorId), + "productid": fmt.Sprintf("0x%s", deviceId), + "hostbus": bus, + "hostaddr": addr, + } +} + +func GetUSBDevQemuOptions(vendorDevId string, addr string) (map[string]interface{}, error) { + parts := strings.Split(vendorDevId, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid vendor_device_id %q", vendorDevId) + } + vendorId := parts[0] + productId := parts[1] + + addrParts := strings.Split(addr, ":") + if len(addrParts) != 2 { + return nil, fmt.Errorf("invalid addr %q", addr) + } + hostBus := addrParts[0] + hostAddr := addrParts[1] + + return getUSBDevQemuOptions(vendorId, productId, hostBus, hostAddr), nil +} + +func (dev *sUSBDevice) GetKernelDriver() (string, error) { + return "", nil +} + +func (dev *sUSBDevice) GetQemuId() (string, error) { + addrParts := strings.Split(dev.dev.Addr, ":") + if len(addrParts) != 2 { + return "", errors.Errorf("Invalid addr %q", dev.dev.Addr) + } + return GetUSBDevId(dev.dev.VendorId, dev.dev.DeviceId, addrParts[0], addrParts[1]), nil +} + +func (dev *sUSBDevice) GetPassthroughCmd(index int) string { + opts, _ := GetUSBDevQemuOptions(dev.dev.GetVendorDeviceId(), dev.dev.Addr) + optsStr := []string{} + for k, v := range opts { + optsStr = append(optsStr, fmt.Sprintf("%s=%s", k, v)) + } + opt := fmt.Sprintf(" -device usb-host,%s", strings.Join(optsStr, ",")) + return opt +} + +func (dev *sUSBDevice) GetHotPlugOptions() ([]*HotPlugOption, error) { + opts, err := GetUSBDevQemuOptions(dev.dev.GetVendorDeviceId(), dev.dev.Addr) + if err != nil { + return nil, errors.Wrap(err, "GetUSBDevQemuOptions") + } + return []*HotPlugOption{ + { + Device: "usb-host", + Options: opts, + }, + }, nil +} + +func (dev *sUSBDevice) GetHotUnplugOptions() ([]*HotUnplugOption, error) { + id, err := dev.GetQemuId() + if err != nil { + return nil, err + } + return []*HotUnplugOption{ + {Id: id}, + }, nil +} + +func getPassthroughUSBs() ([]*sUSBDevice, error) { + ret, err := bashOutput("lsusb") + if err != nil { + return nil, errors.Wrap(err, "execute lsusb") + } + lines := []string{} + for _, l := range ret { + if len(l) != 0 { + lines = append(lines, l) + } + } + + devs, err := parseLsusb(lines) + if err != nil { + return nil, errors.Wrap(err, "parseLsusb") + } + + // fitler linux root hub + retDev := make([]*sUSBDevice, 0) + for _, dev := range devs { + // REF: https://github.com/virt-manager/virt-manager/blob/0038d750c9056ddd63cb48b343e451f8db2746fa/virtinst/nodedev.py#L142 + if isUSBLinuxRootHub(dev.dev.VendorId, dev.dev.DeviceId) { + continue + } + retDev = append(retDev, dev) + } + return retDev, nil +} + +func isUSBLinuxRootHub(vendorId string, deviceId string) bool { + if vendorId == "1d6b" && utils.IsInStringArray(deviceId, []string{"0001", "0002", "0003"}) { + return true + } + return false +} + +func parseLsusb(lines []string) ([]*sUSBDevice, error) { + devs := make([]*sUSBDevice, 0) + for _, line := range lines { + if len(line) == 0 { + continue + } + dev, err := parseLsusbLine(line) + if err != nil { + return nil, errors.Wrapf(err, "parseLsusbLine %q", line) + } + usbDev := newUSBDevice(dev.ToPCIDevice()) + devs = append(devs, usbDev) + } + return devs, nil +} + +var ( + lsusbRegex = `^Bus (?P([0-9]{3})) Device (?P([0-9]{3})): ID (?P([0-9a-z]{4})):(?P([0-9a-z]{4}))\s{0,1}(?P(.*))` +) + +type sLsusbLine struct { + BusId string `json:"bus_id"` + Device string `json:"device"` + VendorId string `json:"vendor_id"` + DeviceId string `json:"device_id"` + Name string `json:"name"` +} + +func parseLsusbLine(line string) (*sLsusbLine, error) { + ret := regutils2.SubGroupMatch(lsusbRegex, line) + dev := new(sLsusbLine) + if err := jsonutils.Marshal(ret).Unmarshal(dev); err != nil { + return nil, err + } + return dev, nil +} + +func (dev *sLsusbLine) ToPCIDevice() *PCIDevice { + return &PCIDevice{ + Addr: fmt.Sprintf("%s:%s", dev.BusId, dev.Device), + VendorId: dev.VendorId, + DeviceId: dev.DeviceId, + ModelName: dev.Name, + } +} diff --git a/pkg/hostman/isolated_device/usb_test.go b/pkg/hostman/isolated_device/usb_test.go new file mode 100644 index 0000000000..39b118d67e --- /dev/null +++ b/pkg/hostman/isolated_device/usb_test.go @@ -0,0 +1,144 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package isolated_device + +import ( + "reflect" + "testing" +) + +func Test_parseLsusbLine(t *testing.T) { + tests := []struct { + name string + lines []string + want []*sLsusbLine + wantErr bool + }{ + { + name: "test", + lines: []string{ + "Bus 004 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub", + "Bus 003 Device 005: ID 13d3:3563 IMC Networks Wireless_Device", + "Bus 003 Device 003: ID 27c6:521d Shenzhen Goodix Technology Co.,Ltd. FingerPrint", + "Bus 003 Device 006: ID 0451:82ff Texas Instruments, Inc.", + "Bus 003 Device 008: ID feed:19c0 YANG HHKB BLE S (USB_DL1K)", + "Bus 003 Device 007: ID 214b:7250 Huasheng Electronics USB2.0 HUB", + "Bus 003 Device 002: ID 0451:8442 Texas Instruments, Inc.", + "Bus 003 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub", + "Bus 002 Device 001: ID 1d6b:0003 Linux Foundation 3.0 root hub", + "Bus 001 Device 004: ID 0b05:193b ASUSTek Computer, Inc. ITE Device(8910)", + "Bus 001 Device 003: ID 0b05:19b6 ASUSTek Computer, Inc. N-KEY Device", + "Bus 001 Device 002: ID 046d:c52f Logitech, Inc. Unifying Receiver", + "Bus 001 Device 001: ID 1d6b:0002 Linux Foundation 2.0 root hub", + }, + want: []*sLsusbLine{ + {BusId: "004", Device: "001", VendorId: "1d6b", DeviceId: "0003", Name: "Linux Foundation 3.0 root hub"}, + {BusId: "003", Device: "005", VendorId: "13d3", DeviceId: "3563", Name: "IMC Networks Wireless_Device"}, + {BusId: "003", Device: "003", VendorId: "27c6", DeviceId: "521d", Name: "Shenzhen Goodix Technology Co.,Ltd. FingerPrint"}, + {BusId: "003", Device: "006", VendorId: "0451", DeviceId: "82ff", Name: "Texas Instruments, Inc."}, + {BusId: "003", Device: "008", VendorId: "feed", DeviceId: "19c0", Name: "YANG HHKB BLE S (USB_DL1K)"}, + {BusId: "003", Device: "007", VendorId: "214b", DeviceId: "7250", Name: "Huasheng Electronics USB2.0 HUB"}, + {BusId: "003", Device: "002", VendorId: "0451", DeviceId: "8442", Name: "Texas Instruments, Inc."}, + {BusId: "003", Device: "001", VendorId: "1d6b", DeviceId: "0002", Name: "Linux Foundation 2.0 root hub"}, + {BusId: "002", Device: "001", VendorId: "1d6b", DeviceId: "0003", Name: "Linux Foundation 3.0 root hub"}, + {BusId: "001", Device: "004", VendorId: "0b05", DeviceId: "193b", Name: "ASUSTek Computer, Inc. ITE Device(8910)"}, + {BusId: "001", Device: "003", VendorId: "0b05", DeviceId: "19b6", Name: "ASUSTek Computer, Inc. N-KEY Device"}, + {BusId: "001", Device: "002", VendorId: "046d", DeviceId: "c52f", Name: "Logitech, Inc. Unifying Receiver"}, + {BusId: "001", Device: "001", VendorId: "1d6b", DeviceId: "0002", Name: "Linux Foundation 2.0 root hub"}, + }, + wantErr: false, + }, + { + name: "with_incomplete_line", + lines: []string{ + "Bus 004 Device 002: ID 17aa:1033", + "Bus 003 Device 002: ID 17aa:1033", + }, + want: []*sLsusbLine{ + {BusId: "004", Device: "002", VendorId: "17aa", DeviceId: "1033", Name: ""}, + {BusId: "003", Device: "002", VendorId: "17aa", DeviceId: "1033", Name: ""}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for i, line := range tt.lines { + got, err := parseLsusbLine(line) + if (err != nil) != tt.wantErr { + t.Errorf("parseLsusb() error = %v, wantErr %v", err, tt.wantErr) + return + } + t.Logf("line result: %#v", got) + + if !reflect.DeepEqual(got, tt.want[i]) { + t.Errorf("parseLsusb() = %v, want %v", got, tt.want[i]) + } + + } + }) + } +} + +func Test_isUSBLinuxRootHub(t *testing.T) { + type args struct { + vendorId string + deviceId string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "1d6b:0001", + args: args{ + "1d6b", + "0001", + }, + want: true, + }, + { + name: "1d6b:0002", + args: args{ + "1d6b", + "0002", + }, + want: true, + }, + { + name: "1d6b:0003", + args: args{ + "1d6b", + "0003", + }, + want: true, + }, + { + name: "1d6b:0004", + args: args{ + "1d6b", + "0004", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isUSBLinuxRootHub(tt.args.vendorId, tt.args.deviceId); got != tt.want { + t.Errorf("isUSBLinuxRootHub() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index ad6b15d15e..bb7eec5d22 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -152,7 +152,8 @@ type SHostOptions struct { DisableKVM bool `help:"force disable KVM" default:"false" json:"disable_kvm"` - DisableGPU bool `help:"force disable GPU" default:"false" json:"disable_gpu"` + DisableGPU bool `help:"force disable GPU detect" default:"false" json:"disable_gpu"` + DisableUSB bool `help:"force disable USB detect" default:"false" json:"disable_usb"` EthtoolEnableGso bool `help:"use ethtool to turn on or off GSO(generic segment offloading)" default:"false" json:"ethtool_enable_gso"`