opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #17776 from wanyaoqi/fix/lvm-disk-deploy

Browse Source 
Fix/lvm disk deploy
This commit is contained in:
Zexi Li
2023-08-18 13:40:01 +08:00
committed by GitHub
parent 3c258def9c 1569dca33b
commit f798d8a2b9
4 changed files with 121 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/hostman/diskutils/kvm.go
+6 -2
View File
@@ -111,7 +111,9 @@ func (d *SKVMGuestDisk) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool
} else {
if partitions[i].Mount() {
support := rootfs.DetectIsUEFISupport(partitions[i])
partitions[i].Umount()
if err := partitions[i].Umount(); err != nil {
log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err)
}
if support {
return true
}
@@ -145,7 +147,9 @@ func (d *SKVMGuestDisk) mountKvmRootfs(readonly bool) (fsdriver.IRootFsDriver, e
return fs, nil
}
errs = append(errs, err)
partitions[i].Umount()
if err := partitions[i].Umount(); err != nil {
log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err)
}
}
}
if len(partitions) == 0 {
pkg/hostman/diskutils/nbd/driver.go
+24 -35
View File
@@ -19,7 +19,6 @@ import (
"io/ioutil"
"path"
"path/filepath"
"runtime/debug"
"strings"
"time"
@@ -42,7 +41,6 @@ type NBDDriver struct {
lvms []*SKVMGuestLVMPartition
imageRootBackFilePath string
imageInfo qemuimg.SImageInfo
acquiredLvm bool
nbdDev string
}
@@ -60,20 +58,21 @@ func init() {
}
func (d *NBDDriver) Connect() error {
pathType := lvmTool.GetPathType(d.rootImagePath())
if pathType == LVM_PATH || pathType == PATH_TYPE_UNKNOWN {
lvmTool.Acquire(d.rootImagePath())
d.acquiredLvm = true
}
d.nbdDev = GetNBDManager().AcquireNbddev()
if len(d.nbdDev) == 0 {
return errors.Errorf("Cannot get nbd device")
}
rootPath := d.rootImagePath()
pathType, lock := lvmTool.Acquire(rootPath)
if pathType != NON_LVM_PATH {
lock.Lock()
defer lock.Unlock()
}
if err := QemuNbdConnect(d.imageInfo, d.nbdDev); err != nil {
return err
}
var tried uint = 0
for len(d.partitions) == 0 && tried < MAX_TRIES {
time.Sleep((1 << tried) * time.Second)
@@ -85,19 +84,21 @@ func (d *NBDDriver) Connect() error {
tried += 1
}
log.Infof("path type %s: %v", d.nbdDev, pathType)
if pathType == LVM_PATH {
if _, err := d.setupLVMS(); err != nil {
return err
}
} else if pathType == PATH_TYPE_UNKNOWN {
hasLVM, err := d.setupLVMS()
log.Infof("%s hasLVM %v err %v", d.nbdDev, hasLVM, err)
if err != nil {
return err
}
// no lvm partition found and has partitions
if !hasLVM && len(d.partitions) > 0 {
d.cacheNonLVMImagePath()
lvmTool.CacheNonLvmImagePath(rootPath)
}
}
return nil
@@ -144,15 +145,6 @@ func (d *NBDDriver) rootImagePath() string {
return d.imageRootBackFilePath
}
func (d *NBDDriver) isNonLvmImagePath() bool {
pathType := lvmTool.GetPathType(d.rootImagePath())
return pathType == NON_LVM_PATH
}
func (d *NBDDriver) cacheNonLVMImagePath() {
lvmTool.CacheNonLvmImagePath(d.rootImagePath())
}
func (d *NBDDriver) setupLVMS() (bool, error) {
// Scan all devices and send the metadata to lvmetad
output, err := procutils.NewCommand("pvscan", "--cache").Output()
@@ -203,26 +195,20 @@ func (d *NBDDriver) findLVMPartitions(partDev string) string {
func (d *NBDDriver) Disconnect() error {
if len(d.nbdDev) > 0 {
defer d.lvmDisconnectNotify()
d.putdownLVMs()
pathType, lock := lvmTool.Acquire(d.rootImagePath())
if pathType != NON_LVM_PATH {
lock.Lock()
defer lock.Unlock()
}
if !d.putdownLVMs() {
return fmt.Errorf("failed putdown lvm devices %s", d.nbdDev)
}
return d.disconnect()
} else {
return nil
}
}
func (d *NBDDriver) lvmDisconnectNotify() {
defer func() {
if r := recover(); r != nil {
log.Errorf("Catch panic on LvmDisconnectNotify %v \n %s", r, debug.Stack())
}
}()
pathType := lvmTool.GetPathType(d.rootImagePath())
if d.acquiredLvm || pathType != NON_LVM_PATH {
lvmTool.Release(d.rootImagePath())
}
}
func (d *NBDDriver) disconnect() error {
if err := QemuNbdDisconnect(d.nbdDev); err != nil {
return err
@@ -233,11 +219,14 @@ func (d *NBDDriver) disconnect() error {
return nil
}
func (d *NBDDriver) putdownLVMs() {
func (d *NBDDriver) putdownLVMs() bool {
for _, lvm := range d.lvms {
lvm.PutdownDevice()
if !lvm.PutdownDevice() {
return false
}
}
d.lvms = []*SKVMGuestLVMPartition{}
return true
}
func (d *NBDDriver) GetPartitions() []fsdriver.IDiskPartition {
pkg/hostman/diskutils/nbd/lvmutils.go
+87 -31
View File
@@ -21,12 +21,14 @@ import (
"regexp"
"strings"
"sync"
"time"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/stringutils"
"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/procutils"
)
@@ -36,50 +38,59 @@ const (
NON_LVM_PATH = 2
)
type SImageProp struct {
HasLVMPartition bool
lock *sync.Mutex
}
type SLVMImageConnectUniqueToolSet struct {
lvms map[string]*sync.Mutex
nonLvms map[string]struct{}
lock *sync.Mutex
*sync.Map
lock *sync.Mutex
}
func NewLVMImageConnectUniqueToolSet() *SLVMImageConnectUniqueToolSet {
return &SLVMImageConnectUniqueToolSet{
lvms: make(map[string]*sync.Mutex),
nonLvms: make(map[string]struct{}),
lock: new(sync.Mutex),
Map: &sync.Map{},
lock: &sync.Mutex{},
}
}
func (s *SLVMImageConnectUniqueToolSet) CacheNonLvmImagePath(imagePath string) {
if im, ok := s.Load(imagePath); ok {
imgProp := im.(*SImageProp)
imgProp.HasLVMPartition = false
}
}
func (s *SLVMImageConnectUniqueToolSet) loadImagePath(imagePath string) (*SImageProp, bool) {
s.lock.Lock()
s.nonLvms[imagePath] = struct{}{}
s.lock.Unlock()
}
func (s *SLVMImageConnectUniqueToolSet) GetPathType(imagePath string) int {
if _, ok := s.nonLvms[imagePath]; ok {
return NON_LVM_PATH
}
if _, ok := s.lvms[imagePath]; ok {
return LVM_PATH
}
return PATH_TYPE_UNKNOWN
}
func (s *SLVMImageConnectUniqueToolSet) Release(imagePath string) {
if _, ok := s.lvms[imagePath]; ok {
s.lvms[imagePath].Unlock()
defer s.lock.Unlock()
im, ok := s.Load(imagePath)
if !ok {
imgProp := &SImageProp{
HasLVMPartition: true, // set has lvm partition default
lock: new(sync.Mutex),
}
s.Store(imagePath, imgProp)
return imgProp, false
} else {
return im.(*SImageProp), ok
}
}
func (s *SLVMImageConnectUniqueToolSet) Acquire(imagePath string) {
s.lock.Lock()
if _, ok := s.lvms[imagePath]; !ok {
s.lvms[imagePath] = new(sync.Mutex)
func (s *SLVMImageConnectUniqueToolSet) Acquire(imagePath string) (int, *sync.Mutex) {
var lock *sync.Mutex
pathType := PATH_TYPE_UNKNOWN
imgProp, ok := s.loadImagePath(imagePath)
if imgProp.HasLVMPartition {
if ok {
pathType = LVM_PATH
}
lock = imgProp.lock
} else {
pathType = NON_LVM_PATH
}
s.lock.Unlock()
s.lvms[imagePath].Lock()
return pathType, lock
}
type SKVMGuestLVMPartition struct {
@@ -205,6 +216,37 @@ func (p *SKVMGuestLVMPartition) FindPartitions() []*kvmpart.SKVMGuestDiskPartiti
return parts
}
func (p *SKVMGuestLVMPartition) UmountPartitions() error {
files, err := ioutil.ReadDir("/dev/" + p.vgname)
if err == nil {
for _, f := range files {
partPath := fmt.Sprintf("/dev/%s/%s", p.vgname, f.Name())
out, err := procutils.NewCommand("umount", partPath).Output()
if err != nil {
log.Errorf("failed umount part %s: %s", partPath, out)
}
}
}
if !os.IsNotExist(err) {
return errors.Errorf("unable to readir /dev/%s: %v", p.vgname, err)
}
lvs, err := p.lvs()
if err != nil {
return errors.Errorf("unable to list lvs: %v", err)
}
for _, lvname := range lvs {
partPath := fmt.Sprintf("/dev/mapper/%s-%s", p.vgname, lvname)
if fileutils2.Exists(partPath) {
out, err := procutils.NewCommand("umount", partPath).Output()
if err != nil {
log.Errorf("failed umount part %s: %s", partPath, out)
}
}
}
return nil
}
var gexp *regexp.Regexp = regexp.MustCompile(`\s+`)
func (p *SKVMGuestLVMPartition) lvs() ([]string, error) {
@@ -230,9 +272,23 @@ func (p *SKVMGuestLVMPartition) lvs() ([]string, error) {
}
func (p *SKVMGuestLVMPartition) PutdownDevice() bool {
if !p.vgActivate(false) {
var deactivate = false
for i := 0; i < 10; i++ {
if !p.vgActivate(false) {
log.Errorf("failed deactivate %s", p.vgname)
if err := p.UmountPartitions(); err != nil {
log.Warningf("failed umount partitions %s", err)
}
time.Sleep(time.Second * 3)
} else {
deactivate = true
break
}
}
if !deactivate {
return false
}
if len(p.originVgname) == 0 || !p.needChangeName {
return true
}
pkg/hostman/guestfs/kvmpart/kvmpart.go
+4 -1
View File
@@ -259,9 +259,11 @@ func (p *SKVMGuestDiskPartition) Umount() error {
var tries = 0
var err error
var out []byte
for tries < 10 {
tries += 1
_, err = procutils.NewCommand("umount", p.mountPath).Output()
log.Infof("umount %s: %s", p.partDev, p.mountPath)
out, err = procutils.NewCommand("umount", p.mountPath).Output()
if err == nil {
if _, err := procutils.NewCommand("blockdev", "--flushbufs", p.partDev).Output(); err != nil {
log.Warningf("blockdev --flushbufs %s error: %v", p.partDev, err)
@@ -272,6 +274,7 @@ func (p *SKVMGuestDiskPartition) Umount() error {
log.Infof("umount %s successfully", p.partDev)
return nil
} else {
log.Warningf("failed umount %s: %s %s", p.partDev, err, out)
time.Sleep(time.Second * 1)
}
}