fix(baremetal-agent): chose root disk by pci path (#23683)

pkg/baremetal/cronjobs.go

@@ -94,7 +94,7 @@ func (job *SStatusProbeJob) Name() string {
 
 func (job *SStatusProbeJob) Do(ctx context.Context, now time.Time) error {
 	if job.baremetal.IsHypervisorHost() {
-		log.Infof("baremetal %q is host, skipping status probe", job.baremetal.GetName())
+		log.Debugf("baremetal %q is host, skipping status probe", job.baremetal.GetName())
 		return nil
 	}
 	bStatus := job.baremetal.GetStatus()

pkg/baremetal/manager.go

@@ -2655,6 +2655,25 @@ func (s *SBaremetalServer) GetMetadata() (*jsonutils.JSONDict, error) {
 }
 
 func (s *SBaremetalServer) GetRootDiskMatcher() (*api.BaremetalRootDiskMatcher, error) {
+	matcher, err := s.getRootDiskMatcher()
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, errors.Wrap(err, "getRootDiskMatcher")
+	}
+	if matcher == nil {
+		matcher = &api.BaremetalRootDiskMatcher{}
+	}
+	rootDiskObj, err := s.GetRootDiskObj()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetRootDiskObj")
+	}
+	pciPath, _ := rootDiskObj.GetString("pci_path")
+	if pciPath != "" {
+		matcher.PCIPath = pciPath
+	}
+	return matcher, nil
+}
+
+func (s *SBaremetalServer) getRootDiskMatcher() (*api.BaremetalRootDiskMatcher, error) {
 	metadata, err := s.GetMetadata()
 	if err != nil {
 		return nil, errors.Wrap(err, "get metadata")
@@ -2692,6 +2711,14 @@ func (s *SBaremetalServer) GetDiskConfig() ([]*api.BaremetalDiskConfig, error) {
 	return baremetal.GetLayoutRaidConfig(layouts), nil
 }
 
+func (s *SBaremetalServer) GetRootDiskObj() (*jsonutils.JSONDict, error) {
+	disks, _ := s.desc.GetArray("disks")
+	if len(disks) == 0 {
+		return nil, errors.Error("Empty disks in desc")
+	}
+	return disks[0].(*jsonutils.JSONDict), nil
+}
+
 func (s *SBaremetalServer) NewConfigedSSHPartitionTool(term *ssh.Client) (*disktool.SSHPartitionTool, error) {
 	raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, false)
 	if err != nil {
@@ -2754,9 +2781,8 @@ func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) (*disktool.SSHPartitio
 	if err != nil {
 		return nil, fmt.Errorf("CalculateLayout: %v", err)
 	}
-	log.Errorf("===layouts: %s", jsonutils.Marshal(layouts).PrettyString())
 	diskConfs := baremetal.GroupLayoutResultsByDriverAdapter(layouts)
-	log.Errorf("===diskConfs: %s", jsonutils.Marshal(diskConfs).PrettyString())
+	log.Errorf("%s layouts: %s, diskConfs: %s", s.GetName(), jsonutils.Marshal(layouts).PrettyString(), jsonutils.Marshal(diskConfs).PrettyString())
 	for _, dConf := range diskConfs {
 		driver := dConf.Driver
 		raidDrv := raiddrivers.GetDriver(driver, term)
@@ -2794,7 +2820,7 @@ func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) (*disktool.SSHPartitio
 	maxTries := 60
 	for tried := 0; !tool.IsAllDisksReady() && tried < maxTries; tried++ {
 		time.Sleep(5 * time.Second)
-		tool.RetrieveDiskInfo()
+		tool.RetrieveDiskInfo(matcher)
 		log.Warningf("disktool not ready string: %s", tool.DebugString())
 	}
 
@@ -2848,10 +2874,10 @@ func (s *SBaremetalServer) doCreateRoot(term *ssh.Client, devName string, disabl
 	return nil
 }
 
-func (s *SBaremetalServer) DoPartitionDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) ([]*disktool.Partition, error) {
+func (s *SBaremetalServer) DoPartitionDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) (*disktool.DiskPartitions, []*disktool.Partition, error) {
 	raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, false)
 	if err != nil {
-		return nil, err
+		return nil, nil, errors.Wrap(err, "DetectStorageInfo")
 	}
 	storages := make([]*baremetal.BaremetalStorage, 0)
 	storages = append(storages, raid...)
@@ -2873,33 +2899,34 @@ func (s *SBaremetalServer) DoPartitionDisk(tool *disktool.SSHPartitionTool, term
 
 	disks, _ := s.desc.GetArray("disks")
 	if len(disks) == 0 {
-		return nil, errors.Error("Empty disks in desc")
+		return nil, nil, errors.Error("Empty disks in desc")
 	}
 
 	rootImageId := s.GetRootTemplateId()
 	diskOffset := 0
+	rootDisk := tool.GetRootDisk()
 	if len(rootImageId) > 0 {
-		rootDisk := disks[0]
-		rootSize, _ := rootDisk.Int("size")
-		err = s.doCreateRoot(term, tool.GetRootDisk().GetDevName(), disableImageCache)
+		rootDiskObj := disks[0]
+		rootSize, _ := rootDiskObj.Int("size")
+		err = s.doCreateRoot(term, rootDisk.GetDevName(), disableImageCache)
 		if err != nil {
-			return nil, errors.Wrap(err, "Failed to create root")
+			return rootDisk, nil, errors.Wrap(err, "Failed to create root")
 		}
 		tool.RetrievePartitionInfo()
 		parts := tool.GetPartitions()
 		if len(parts) == 0 {
-			return nil, errors.Error("Root disk create failed, no partitions")
+			return rootDisk, nil, errors.Error("Root disk create failed, no partitions")
 		}
 		log.Infof("Resize root to %d MB", rootSize)
 		if err := tool.ResizePartition(0, rootSize); err != nil {
-			return nil, errors.Wrapf(err, "Fail to resize root to %d", rootSize)
+			return rootDisk, nil, errors.Wrapf(err, "Fail to resize root to %d", rootSize)
 		}
 		diskOffset = 1
 	} else {
 		tool.RetrievePartitionInfo()
 		parts := tool.GetPartitions()
 		if len(parts) > 0 {
-			return nil, errors.Error("should no partition!!!")
+			return rootDisk, nil, errors.Error("should no partition!!!")
 		}
 	}
 
@@ -2914,16 +2941,16 @@ func (s *SBaremetalServer) DoPartitionDisk(tool *disktool.SSHPartitionTool, term
 			driver, _ := disk.GetString("driver")
 			log.Infof("Create partition %d %s", sz, fs)
 			if err := tool.CreatePartition(-1, sz, fs, true, driver, uuid); err != nil {
-				return nil, errors.Wrapf(err, "Fail to create disk %s", disk.String())
+				return rootDisk, nil, errors.Wrapf(err, "Fail to create disk %s", disk.String())
 			}
 		}
 	}
 	log.Infof("Finish create partitions")
 
-	return tool.GetPartitions(), nil
+	return rootDisk, tool.GetPartitions(), nil
 }
 
-func (s *SBaremetalServer) DoRebuildRootDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) ([]*disktool.Partition, error) {
+func (s *SBaremetalServer) DoRebuildRootDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) (*disktool.DiskPartitions, []*disktool.Partition, error) {
 	// raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, false)
 	// if err != nil {
 	// 	return nil, err
@@ -2948,7 +2975,7 @@ func (s *SBaremetalServer) DoRebuildRootDisk(tool *disktool.SSHPartitionTool, te
 
 	disks, _ := s.desc.GetArray("disks")
 	if len(disks) == 0 {
-		return nil, fmt.Errorf("Empty disks in desc")
+		return nil, nil, fmt.Errorf("Empty disks in desc")
 	}
 
 	rootDisk := disks[0]
@@ -2956,16 +2983,16 @@ func (s *SBaremetalServer) DoRebuildRootDisk(tool *disktool.SSHPartitionTool, te
 	rd := tool.GetRootDisk()
 	err := s.doCreateRoot(term, rd.GetDevName(), disableImageCache)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to create root: %v", err)
+		return rd, nil, fmt.Errorf("Failed to create root: %v", err)
 	}
 	tool.RetrievePartitionInfo()
 	if err := rd.ReInitInfo(); err != nil {
-		return nil, errors.Wrap(err, "Reinit root disk after create root")
+		return rd, nil, errors.Wrap(err, "Reinit root disk after create root")
 	}
 
 	log.Infof("Resize root to %d MB", rootSize)
 	if err := rd.ResizePartition(rootSize); err != nil {
-		return nil, fmt.Errorf("Fail to resize root to %d, err: %v", rootSize, err)
+		return rd, nil, fmt.Errorf("Fail to resize root to %d, err: %v", rootSize, err)
 	}
 	if len(disks) > 1 {
 		for _, disk := range disks[1:] {
@@ -2992,10 +3019,10 @@ func (s *SBaremetalServer) DoRebuildRootDisk(tool *disktool.SSHPartitionTool, te
 	for _, d := range restDisks {
 		parts = append(parts, d.GetPartitions()...)
 	}
-	return parts, nil
+	return rd, parts, nil
 }
 
-func (s *SBaremetalServer) SyncPartitionSize(term *ssh.Client, parts []*disktool.Partition) ([]jsonutils.JSONObject, error) {
+func (s *SBaremetalServer) SyncPartitionSize(term *ssh.Client, rootDisk *disktool.DiskPartitions, parts []*disktool.Partition) ([]jsonutils.JSONObject, error) {
 	disks, _ := s.desc.GetArray("disks")
 
 	// calculate root partitions count
@@ -3007,18 +3034,24 @@ func (s *SBaremetalServer) SyncPartitionSize(term *ssh.Client, parts []*disktool
 	rootParts := parts[0:rootPartsCnt]
 	dataParts := parts[rootPartsCnt:]
 	idx := 0
+
+	// set root disk attributes that returns to region service
 	size := (rootParts[len(rootParts)-1].GetEnd() + 1) * 512 / 1024 / 1024
-	disks[idx].(*jsonutils.JSONDict).Set("size", jsonutils.NewInt(int64(size)))
+	rootDiskObj := disks[idx].(*jsonutils.JSONDict)
+	rootDiskObj.Set("size", jsonutils.NewInt(int64(size)))
+	rootDiskObj.Set("pci_path", jsonutils.NewString(rootDisk.GetPCIPath()))
+
 	idx += 1
 	for _, p := range dataParts {
 		sizeMB, err := p.GetSizeMB()
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "GetSizeMB")
 		}
 		disks[idx].(*jsonutils.JSONDict).Set("size", jsonutils.NewInt(int64(sizeMB)))
 		disks[idx].(*jsonutils.JSONDict).Set("dev", jsonutils.NewString(p.GetDev()))
 		idx++
 	}
+	s.desc.Set("disks", jsonutils.NewArray(disks...))
 	return disks, nil
 }
 

pkg/baremetal/tasks/create.go

@@ -64,12 +64,13 @@ func (self *SBaremetalServerCreateTask) DoDeploys(ctx context.Context, term *ssh
 		return nil, self.onError(ctx, term, err)
 	}
 	time.Sleep(2 * time.Second)
-	parts, err := self.Baremetal.GetServer().DoPartitionDisk(tool, term, self.IsDisableImageCache())
+	rootDisk, parts, err := self.Baremetal.GetServer().DoPartitionDisk(tool, term, self.IsDisableImageCache())
 	if err != nil {
 		return nil, self.onError(ctx, term, err)
 	}
 	data := jsonutils.NewDict()
-	disks, err := self.Baremetal.GetServer().SyncPartitionSize(term, parts)
+
+	disks, err := self.Baremetal.GetServer().SyncPartitionSize(term, rootDisk, parts)
 	if err != nil {
 		return nil, self.onError(ctx, term, err)
 	}

pkg/baremetal/tasks/rebuild.go

@@ -58,11 +58,11 @@ func (self *SBaremetalServerRebuildTask) DoDeploys(ctx context.Context, term *ss
 	if err != nil {
 		return nil, errors.Wrap(err, "NewConfigedSSHPartitionTool")
 	}
-	parts, err := self.Baremetal.GetServer().DoRebuildRootDisk(tool, term, self.IsDisableImageCache())
+	rootDisk, parts, err := self.Baremetal.GetServer().DoRebuildRootDisk(tool, term, self.IsDisableImageCache())
 	if err != nil {
 		return nil, fmt.Errorf("Rebuild root disk: %v", err)
 	}
-	disks, err := self.Baremetal.GetServer().SyncPartitionSize(term, parts)
+	disks, err := self.Baremetal.GetServer().SyncPartitionSize(term, rootDisk, parts)
 	if err != nil {
 		return nil, fmt.Errorf("SyncPartitionSize: %v", err)
 	}

pkg/baremetal/types/types.go

@@ -31,10 +31,10 @@ type IBaremetalServer interface {
 	DoDiskUnconfig(term *ssh.Client) error
 	DoDiskConfig(term *ssh.Client) (*disktool.SSHPartitionTool, error)
 	DoEraseDisk(term *ssh.Client) error
-	DoPartitionDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) ([]*disktool.Partition, error)
+	DoPartitionDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) (*disktool.DiskPartitions, []*disktool.Partition, error)
 	NewConfigedSSHPartitionTool(term *ssh.Client) (*disktool.SSHPartitionTool, error)
-	DoRebuildRootDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) ([]*disktool.Partition, error)
-	SyncPartitionSize(term *ssh.Client, parts []*disktool.Partition) ([]jsonutils.JSONObject, error)
+	DoRebuildRootDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) (*disktool.DiskPartitions, []*disktool.Partition, error)
+	SyncPartitionSize(term *ssh.Client, rootDisk *disktool.DiskPartitions, parts []*disktool.Partition) ([]jsonutils.JSONObject, error)
 	DoDeploy(tool *disktool.SSHPartitionTool, term *ssh.Client, data jsonutils.JSONObject, isInit bool) (jsonutils.JSONObject, error)
 	SaveDesc(desc jsonutils.JSONObject) error
 	GetNics() []types.SServerNic

pkg/baremetal/utils/disktool/disktool.go

@@ -19,6 +19,7 @@ import (
 	"math"
 	"strings"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
@@ -224,6 +225,7 @@ type DiskPartitions struct {
 	rotate     bool
 	desc       string
 	label      string
+	pciPath    string
 	partitions []*Partition
 }
 
@@ -253,7 +255,13 @@ func (p *DiskPartitions) GetDev() string {
 	return p.dev
 }
 
-func (p *DiskPartitions) SetInfo(info *types.SDiskInfo) *DiskPartitions {
+func getPCIPathPrefix(input string) string {
+	input = strings.TrimSpace(input)
+	parts := strings.Split(input, "/")
+	return strings.Join(parts[0:len(parts)-2], "/")
+}
+
+func (p *DiskPartitions) SetInfo(info *types.SDiskInfo) (*DiskPartitions, error) {
 	p.dev = fmt.Sprintf("/dev/%s", info.Dev)
 	p.devName = info.Dev
 	p.sectors = info.Sector
@@ -262,7 +270,17 @@ func (p *DiskPartitions) SetInfo(info *types.SDiskInfo) *DiskPartitions {
 	if p.blockSize == 4096 {
 		p.sectors = (p.sectors >> 3)
 	}
-	return p
+
+	// /sys/block/sda => /sys/devices/pci0000:00/0000:00:04.0/virtio1/host0/target0:0:0/0:0:0:0/block/sda
+	// /sys/block/nvme1n1 => /sys/devices/pci0000:00/0000:00:1a.0/0000:03:00.0/nvme/nvme1/nvme1n1
+	outputs, err := p.Run(fmt.Sprintf("readlink -f /sys/block/%s", info.Dev))
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to read device %s pci path", p.dev)
+	}
+	if len(outputs) > 0 {
+		p.pciPath = getPCIPathPrefix(outputs[0])
+	}
+	return p, nil
 }
 
 func (p *DiskPartitions) ReInitInfo() error {
@@ -273,7 +291,9 @@ func (p *DiskPartitions) ReInitInfo() error {
 	}
 	for _, disk := range sysutils.ParseDiskInfo(lines, p.driver) {
 		if disk.Dev == p.GetDevName() {
-			p.SetInfo(disk)
+			if _, err := p.SetInfo(disk); err != nil {
+				return errors.Wrapf(err, "set info of disk %v", disk)
+			}
 		}
 	}
 	return p.RetrievePartitionInfo()
@@ -326,6 +346,10 @@ func (ps *DiskPartitions) GetDevName() string {
 	return devName
 }
 
+func (ps *DiskPartitions) GetPCIPath() string {
+	return ps.pciPath
+}
+
 func (ps *DiskPartitions) RetrievePartitionInfo() error {
 	ps.partitions = make([]*Partition, 0)
 	cmd := []string{"parted", "-s", ps.dev, "--", "unit", "s", "print"}
@@ -664,7 +688,7 @@ func (tool *PartitionTool) parseLsDisk(lines []string, driver string) {
 	}
 }
 
-func (tool *PartitionTool) FetchDiskConfs(diskConfs []baremetal.DiskConfiguration, rootMatcher *api.BaremetalRootDiskMatcher) *PartitionTool {
+func (tool *PartitionTool) FetchDiskConfs(diskConfs []baremetal.DiskConfiguration) *PartitionTool {
 	for _, d := range diskConfs {
 		disk := newDiskPartitions(d.Driver, d.Adapter, d.RaidConfig, d.Size, d.Block, d.DiskType, tool)
 		tool.disks = append(tool.disks, disk)
@@ -681,16 +705,10 @@ func (tool *PartitionTool) FetchDiskConfs(diskConfs []baremetal.DiskConfiguratio
 		}
 		tool.diskTable[key] = append(tool.diskTable[key], disk)
 	}
-	// reorder tool.disks
-	if rootMatcher != nil {
-		tool.reorderRootDisk(rootMatcher)
-	}
 	return tool
 }
 
 func (tool *PartitionTool) reorderRootDisk(matcher *api.BaremetalRootDiskMatcher) {
-	var rootDiskIdx = 0
-
 	isDiskMatch := func(disk *DiskPartitions, matcher *api.BaremetalRootDiskMatcher) bool {
 		if matcher.Device != "" {
 			if disk.dev == matcher.Device {
@@ -710,16 +728,25 @@ func (tool *PartitionTool) reorderRootDisk(matcher *api.BaremetalRootDiskMatcher
 				return true
 			}
 		}
+		if matcher.PCIPath != "" {
+			if disk.pciPath == matcher.PCIPath {
+				return true
+			}
+		}
 		return false
 	}
 
+	var rootDiskStr string
+	var rootDiskIdx = 0
+
 	for idx, disk := range tool.disks {
 		if isDiskMatch(disk, matcher) {
 			rootDiskIdx = idx
+			rootDiskStr = disk.String()
 			break
 		}
 	}
-	log.Infof("Select %d as root disk", rootDiskIdx)
+	log.Infof("Select %d %q as root disk by matcher: %s", rootDiskIdx, rootDiskStr, jsonutils.Marshal(matcher))
 	newDisks := make([]*DiskPartitions, 0)
 	newDisks = append(newDisks, tool.disks[rootDiskIdx])
 	for idx := range tool.disks {
@@ -741,7 +768,7 @@ func (tool *PartitionTool) IsAllDisksReady() bool {
 	return true
 }
 
-func (tool *PartitionTool) RetrieveDiskInfo() error {
+func (tool *PartitionTool) RetrieveDiskInfo(rootMatcher *api.BaremetalRootDiskMatcher) error {
 	for _, driver := range []string{RAID_DRVIER, NONRAID_DRIVER, PCIE_DRIVER} {
 		cmd := fmt.Sprintf("/lib/mos/lsdisk --%s", driver)
 		ret, err := tool.Run(cmd)
@@ -750,6 +777,10 @@ func (tool *PartitionTool) RetrieveDiskInfo() error {
 		}
 		tool.parseLsDisk(ret, driver)
 	}
+	// reorder tool.disks
+	if rootMatcher != nil {
+		tool.reorderRootDisk(rootMatcher)
+	}
 	return nil
 }
 
@@ -837,8 +868,8 @@ func newSSHPartitionTool(term *ssh.Client) *SSHPartitionTool {
 
 func NewSSHPartitionTool(term *ssh.Client, layouts []baremetal.Layout, rootMatcher *api.BaremetalRootDiskMatcher) (*SSHPartitionTool, error) {
 	tool := newSSHPartitionTool(term)
-	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts), rootMatcher)
-	if err := tool.RetrieveDiskInfo(); err != nil {
+	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))
+	if err := tool.RetrieveDiskInfo(rootMatcher); err != nil {
 		return nil, errors.Wrapf(err, "RetrieveDiskInfo")
 	}
 	return tool, nil

pkg/baremetal/utils/disktool/disktool_test.go

@@ -14,6 +14,8 @@
 
 package disktool
 
+import "testing"
+
 // TODO: use mock ssh server backend test disktool
 /*
 import (
@@ -64,3 +66,26 @@ func TestSSHCreate(t *testing.T) {
 		t.Errorf("Failed to resize fs: %v", err)
 	}
 }*/
+
+func Test_getPCIPathPrefix(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{
+			input: "/sys/devices/pci0000:00/0000:00:02.2/0000:02:00.0/host0/target0:1:0/0:1:0:0/block/sda",
+			want:  "/sys/devices/pci0000:00/0000:00:02.2/0000:02:00.0/host0/target0:1:0/0:1:0:0",
+		},
+		{
+			input: "/sys/devices/pci0000:00/0000:00:06.0/0000:02:00.0/nvme/nvme0/nvme0n1",
+			want:  "/sys/devices/pci0000:00/0000:00:06.0/0000:02:00.0/nvme",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			if got := getPCIPathPrefix(tt.input); got != tt.want {
+				t.Errorf("getPCIPathPrefix() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}