diff --git a/pkg/hostman/diskutils/kvm.go b/pkg/hostman/diskutils/kvm.go index c0edbbc9ae..8834a53482 100644 --- a/pkg/hostman/diskutils/kvm.go +++ b/pkg/hostman/diskutils/kvm.go @@ -111,7 +111,9 @@ func (d *SKVMGuestDisk) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool } else { if partitions[i].Mount() { support := rootfs.DetectIsUEFISupport(partitions[i]) - partitions[i].Umount() + if err := partitions[i].Umount(); err != nil { + log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err) + } if support { return true } @@ -145,7 +147,9 @@ func (d *SKVMGuestDisk) mountKvmRootfs(readonly bool) (fsdriver.IRootFsDriver, e return fs, nil } errs = append(errs, err) - partitions[i].Umount() + if err := partitions[i].Umount(); err != nil { + log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err) + } } } if len(partitions) == 0 { diff --git a/pkg/hostman/diskutils/nbd/driver.go b/pkg/hostman/diskutils/nbd/driver.go index 51fb2d42dd..42b6fb4750 100644 --- a/pkg/hostman/diskutils/nbd/driver.go +++ b/pkg/hostman/diskutils/nbd/driver.go @@ -19,7 +19,6 @@ import ( "io/ioutil" "path" "path/filepath" - "runtime/debug" "strings" "time" @@ -42,7 +41,6 @@ type NBDDriver struct { lvms []*SKVMGuestLVMPartition imageRootBackFilePath string imageInfo qemuimg.SImageInfo - acquiredLvm bool nbdDev string } @@ -60,20 +58,21 @@ func init() { } func (d *NBDDriver) Connect() error { - pathType := lvmTool.GetPathType(d.rootImagePath()) - if pathType == LVM_PATH || pathType == PATH_TYPE_UNKNOWN { - lvmTool.Acquire(d.rootImagePath()) - d.acquiredLvm = true - } - d.nbdDev = GetNBDManager().AcquireNbddev() if len(d.nbdDev) == 0 { return errors.Errorf("Cannot get nbd device") } + + rootPath := d.rootImagePath() + pathType, lock := lvmTool.Acquire(rootPath) + if pathType != NON_LVM_PATH { + lock.Lock() + defer lock.Unlock() + } + if err := QemuNbdConnect(d.imageInfo, d.nbdDev); err != nil { return err } - var tried uint = 0 for len(d.partitions) == 0 && tried < MAX_TRIES { time.Sleep((1 << tried) * time.Second) @@ -85,19 +84,21 @@ func (d *NBDDriver) Connect() error { tried += 1 } + log.Infof("path type %s: %v", d.nbdDev, pathType) if pathType == LVM_PATH { if _, err := d.setupLVMS(); err != nil { return err } } else if pathType == PATH_TYPE_UNKNOWN { hasLVM, err := d.setupLVMS() + log.Infof("%s hasLVM %v err %v", d.nbdDev, hasLVM, err) if err != nil { return err } // no lvm partition found and has partitions if !hasLVM && len(d.partitions) > 0 { - d.cacheNonLVMImagePath() + lvmTool.CacheNonLvmImagePath(rootPath) } } return nil @@ -144,15 +145,6 @@ func (d *NBDDriver) rootImagePath() string { return d.imageRootBackFilePath } -func (d *NBDDriver) isNonLvmImagePath() bool { - pathType := lvmTool.GetPathType(d.rootImagePath()) - return pathType == NON_LVM_PATH -} - -func (d *NBDDriver) cacheNonLVMImagePath() { - lvmTool.CacheNonLvmImagePath(d.rootImagePath()) -} - func (d *NBDDriver) setupLVMS() (bool, error) { // Scan all devices and send the metadata to lvmetad output, err := procutils.NewCommand("pvscan", "--cache").Output() @@ -203,26 +195,20 @@ func (d *NBDDriver) findLVMPartitions(partDev string) string { func (d *NBDDriver) Disconnect() error { if len(d.nbdDev) > 0 { - defer d.lvmDisconnectNotify() - d.putdownLVMs() + pathType, lock := lvmTool.Acquire(d.rootImagePath()) + if pathType != NON_LVM_PATH { + lock.Lock() + defer lock.Unlock() + } + if !d.putdownLVMs() { + return fmt.Errorf("failed putdown lvm devices %s", d.nbdDev) + } return d.disconnect() } else { return nil } } -func (d *NBDDriver) lvmDisconnectNotify() { - defer func() { - if r := recover(); r != nil { - log.Errorf("Catch panic on LvmDisconnectNotify %v \n %s", r, debug.Stack()) - } - }() - pathType := lvmTool.GetPathType(d.rootImagePath()) - if d.acquiredLvm || pathType != NON_LVM_PATH { - lvmTool.Release(d.rootImagePath()) - } -} - func (d *NBDDriver) disconnect() error { if err := QemuNbdDisconnect(d.nbdDev); err != nil { return err @@ -233,11 +219,14 @@ func (d *NBDDriver) disconnect() error { return nil } -func (d *NBDDriver) putdownLVMs() { +func (d *NBDDriver) putdownLVMs() bool { for _, lvm := range d.lvms { - lvm.PutdownDevice() + if !lvm.PutdownDevice() { + return false + } } d.lvms = []*SKVMGuestLVMPartition{} + return true } func (d *NBDDriver) GetPartitions() []fsdriver.IDiskPartition { diff --git a/pkg/hostman/diskutils/nbd/lvmutils.go b/pkg/hostman/diskutils/nbd/lvmutils.go index c395f073cf..fe9810ce5a 100644 --- a/pkg/hostman/diskutils/nbd/lvmutils.go +++ b/pkg/hostman/diskutils/nbd/lvmutils.go @@ -21,12 +21,14 @@ import ( "regexp" "strings" "sync" + "time" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/stringutils" "yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart" + "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" ) @@ -36,50 +38,59 @@ const ( NON_LVM_PATH = 2 ) +type SImageProp struct { + HasLVMPartition bool + lock *sync.Mutex +} + type SLVMImageConnectUniqueToolSet struct { - lvms map[string]*sync.Mutex - nonLvms map[string]struct{} - lock *sync.Mutex + *sync.Map + lock *sync.Mutex } func NewLVMImageConnectUniqueToolSet() *SLVMImageConnectUniqueToolSet { return &SLVMImageConnectUniqueToolSet{ - lvms: make(map[string]*sync.Mutex), - nonLvms: make(map[string]struct{}), - lock: new(sync.Mutex), + Map: &sync.Map{}, + lock: &sync.Mutex{}, } } func (s *SLVMImageConnectUniqueToolSet) CacheNonLvmImagePath(imagePath string) { + if im, ok := s.Load(imagePath); ok { + imgProp := im.(*SImageProp) + imgProp.HasLVMPartition = false + } +} + +func (s *SLVMImageConnectUniqueToolSet) loadImagePath(imagePath string) (*SImageProp, bool) { s.lock.Lock() - s.nonLvms[imagePath] = struct{}{} - s.lock.Unlock() -} - -func (s *SLVMImageConnectUniqueToolSet) GetPathType(imagePath string) int { - if _, ok := s.nonLvms[imagePath]; ok { - return NON_LVM_PATH - } - if _, ok := s.lvms[imagePath]; ok { - return LVM_PATH - } - return PATH_TYPE_UNKNOWN -} - -func (s *SLVMImageConnectUniqueToolSet) Release(imagePath string) { - if _, ok := s.lvms[imagePath]; ok { - s.lvms[imagePath].Unlock() + defer s.lock.Unlock() + im, ok := s.Load(imagePath) + if !ok { + imgProp := &SImageProp{ + HasLVMPartition: true, // set has lvm partition default + lock: new(sync.Mutex), + } + s.Store(imagePath, imgProp) + return imgProp, false + } else { + return im.(*SImageProp), ok } } -func (s *SLVMImageConnectUniqueToolSet) Acquire(imagePath string) { - s.lock.Lock() - if _, ok := s.lvms[imagePath]; !ok { - s.lvms[imagePath] = new(sync.Mutex) +func (s *SLVMImageConnectUniqueToolSet) Acquire(imagePath string) (int, *sync.Mutex) { + var lock *sync.Mutex + pathType := PATH_TYPE_UNKNOWN + imgProp, ok := s.loadImagePath(imagePath) + if imgProp.HasLVMPartition { + if ok { + pathType = LVM_PATH + } + lock = imgProp.lock + } else { + pathType = NON_LVM_PATH } - s.lock.Unlock() - - s.lvms[imagePath].Lock() + return pathType, lock } type SKVMGuestLVMPartition struct { @@ -205,6 +216,37 @@ func (p *SKVMGuestLVMPartition) FindPartitions() []*kvmpart.SKVMGuestDiskPartiti return parts } +func (p *SKVMGuestLVMPartition) UmountPartitions() error { + files, err := ioutil.ReadDir("/dev/" + p.vgname) + if err == nil { + for _, f := range files { + partPath := fmt.Sprintf("/dev/%s/%s", p.vgname, f.Name()) + out, err := procutils.NewCommand("umount", partPath).Output() + if err != nil { + log.Errorf("failed umount part %s: %s", partPath, out) + } + } + } + + if !os.IsNotExist(err) { + return errors.Errorf("unable to readir /dev/%s: %v", p.vgname, err) + } + lvs, err := p.lvs() + if err != nil { + return errors.Errorf("unable to list lvs: %v", err) + } + for _, lvname := range lvs { + partPath := fmt.Sprintf("/dev/mapper/%s-%s", p.vgname, lvname) + if fileutils2.Exists(partPath) { + out, err := procutils.NewCommand("umount", partPath).Output() + if err != nil { + log.Errorf("failed umount part %s: %s", partPath, out) + } + } + } + return nil +} + var gexp *regexp.Regexp = regexp.MustCompile(`\s+`) func (p *SKVMGuestLVMPartition) lvs() ([]string, error) { @@ -230,9 +272,23 @@ func (p *SKVMGuestLVMPartition) lvs() ([]string, error) { } func (p *SKVMGuestLVMPartition) PutdownDevice() bool { - if !p.vgActivate(false) { + var deactivate = false + for i := 0; i < 10; i++ { + if !p.vgActivate(false) { + log.Errorf("failed deactivate %s", p.vgname) + if err := p.UmountPartitions(); err != nil { + log.Warningf("failed umount partitions %s", err) + } + time.Sleep(time.Second * 3) + } else { + deactivate = true + break + } + } + if !deactivate { return false } + if len(p.originVgname) == 0 || !p.needChangeName { return true } diff --git a/pkg/hostman/guestfs/kvmpart/kvmpart.go b/pkg/hostman/guestfs/kvmpart/kvmpart.go index 60ba57e647..aac3f57740 100644 --- a/pkg/hostman/guestfs/kvmpart/kvmpart.go +++ b/pkg/hostman/guestfs/kvmpart/kvmpart.go @@ -259,9 +259,11 @@ func (p *SKVMGuestDiskPartition) Umount() error { var tries = 0 var err error + var out []byte for tries < 10 { tries += 1 - _, err = procutils.NewCommand("umount", p.mountPath).Output() + log.Infof("umount %s: %s", p.partDev, p.mountPath) + out, err = procutils.NewCommand("umount", p.mountPath).Output() if err == nil { if _, err := procutils.NewCommand("blockdev", "--flushbufs", p.partDev).Output(); err != nil { log.Warningf("blockdev --flushbufs %s error: %v", p.partDev, err) @@ -272,6 +274,7 @@ func (p *SKVMGuestDiskPartition) Umount() error { log.Infof("umount %s successfully", p.partDev) return nil } else { + log.Warningf("failed umount %s: %s %s", p.partDev, err, out) time.Sleep(time.Second * 1) } }