Merge pull request #10223 from swordqiu/hotfix/qj-host-deployer-mount-retry

fix(host-deployer): mount may fail to lock /etc/mtab, add retrier

pkg/hostman/diskutils/fsutils/fsutils.go

@@ -274,7 +274,7 @@ func FsckExtFs(fpath string) bool {
 	log.Debugf("Exec command: %v", []string{"e2fsck", "-f", "-p", fpath})
 	cmd := procutils.NewCommand("e2fsck", "-f", "-p", fpath)
 	if err := cmd.Start(); err != nil {
-		log.Errorln(err)
+		log.Errorf("e2fsck start failed: %s", err)
 		return false
 	} else {
 		err = cmd.Wait()
@@ -292,11 +292,17 @@ func FsckExtFs(fpath string) bool {
 	}
 }
 
+// https://bugs.launchpad.net/ubuntu/+source/xfsprogs/+bug/1718244
+// use xfs_repair -n instead
 func FsckXfsFs(fpath string) bool {
 	if output, err := procutils.NewCommand("xfs_check", fpath).Output(); err != nil {
-		log.Errorf("xfs_check failed: %s, %s", err, output)
-		procutils.NewCommand("xfs_repair", fpath).Output()
-		return false
+		log.Errorf("xfs_check failed: %s, %s, try xfs_repair -n <dev> instead", err, output)
+		if output, err := procutils.NewCommand("xfs_repair", "-n", fpath).Output(); err != nil {
+			log.Errorf("xfs_repair -n dev failed: %s, %s", err, output)
+			// repair the xfs
+			procutils.NewCommand("xfs_repair", fpath).Output()
+			return false
+		}
 	}
 	return true
 }

pkg/hostman/guestfs/kvmpart/kvmpart.go

@@ -15,7 +15,9 @@
 package kvmpart
 
 import (
+	"context"
 	"fmt"
+	"math/rand"
 	"os"
 	"strings"
 	"time"
@@ -173,8 +175,22 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
 			}()
 		}
 	}
-	output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output()
-	return errors.Wrapf(err, "mount failed: %s", output)
+
+	retrier := func(utils.FibonacciRetrier) (bool, error) {
+		output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output()
+		if err == nil {
+			return true, nil
+		} else {
+			log.Errorf("mount fail: %s %s", err, output)
+			time.Sleep(time.Millisecond * time.Duration(100+rand.Intn(400)))
+			return false, errors.Wrap(err, "")
+		}
+	}
+	_, err = utils.NewFibonacciRetrierMaxTries(3, retrier).Start(context.Background())
+	if err != nil {
+		return errors.Wrap(err, "mount failed")
+	}
+	return nil // errors.Wrapf(err, "mount failed: %s", output)
 }
 
 func (p *SKVMGuestDiskPartition) fsck() error {