Automatic merge from release/2.6.0 -> release/2.7.0

* commit 'b43aac4d010722337b45c148d20091eeff26a070':
  fix: isActive error message include filename
  fix: not set converting status
  fix: glance image convert consume too much IO, use ionice -c idle to reduce its IO impact

pkg/image/models/images.go

@@ -37,10 +37,10 @@ type TImageType string
 const (
 	// https://docs.openstack.org/glance/pike/user/statuses.html
 	//
-	IMAGE_STATUS_QUEUED     = "queued"
-	IMAGE_STATUS_SAVING     = "saving"
-	IMAGE_STATUS_ACTIVE     = "active"
-	IMAGE_STATUS_CONVERTING = "converting"
+	IMAGE_STATUS_QUEUED = "queued"
+	IMAGE_STATUS_SAVING = "saving"
+	IMAGE_STATUS_ACTIVE = "active"
+	// IMAGE_STATUS_CONVERTING = "converting"
 
 	IMAGE_STATUS_DEACTIVATED    = "deactivated"
 	IMAGE_STATUS_KILLED         = "killed"
@@ -550,7 +550,10 @@ func (self *SImage) CustomizeDelete(ctx context.Context, userCred mcclient.Token
 		overridePendingDelete = jsonutils.QueryBoolean(query, "override_pending_delete", false)
 		purge = jsonutils.QueryBoolean(query, "purge", false)
 	}
-	if self.Status != IMAGE_STATUS_ACTIVE && self.Status != IMAGE_STATUS_CONVERTING {
+	if utils.IsInStringArray(self.Status, []string{
+		IMAGE_STATUS_KILLED,
+		IMAGE_STATUS_QUEUED,
+	}) {
 		overridePendingDelete = true
 	}
 	return self.startDeleteImageTask(ctx, userCred, "", purge, overridePendingDelete)
@@ -885,7 +888,7 @@ func (self *SImage) getLocalLocation() string {
 }
 
 func (self *SImage) getQemuImage() (*qemuimg.SQemuImage, error) {
-	return qemuimg.NewQemuImage(self.getLocalLocation())
+	return qemuimg.NewQemuImageWithIOLevel(self.getLocalLocation(), qemuimg.IONiceIdle)
 }
 
 func (self *SImage) StopTorrents() {
@@ -968,31 +971,31 @@ func (manager *SImageManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ
 
 func isActive(localPath string, size int64, chksum string, fastHash string, useFastHash bool) bool {
 	if len(localPath) == 0 || !fileutils2.Exists(localPath) {
-		log.Errorf("invalid file")
+		log.Errorf("invalid file: %s", localPath)
 		return false
 	}
 	if size != fileutils2.FileSize(localPath) {
-		log.Errorf("size mistmatch")
+		log.Errorf("size mistmatch: %s", localPath)
 		return false
 	}
 	if useFastHash && len(fastHash) > 0 {
 		fhash, err := fileutils2.FastCheckSum(localPath)
 		if err != nil {
-			log.Errorf("IsActive fastChecksum fail %s", err)
+			log.Errorf("IsActive fastChecksum fail %s for %s", err, localPath)
 			return false
 		}
 		if fastHash != fhash {
-			log.Errorf("IsActive fastChecksum mismatch")
+			log.Errorf("IsActive fastChecksum mismatch for %s", localPath)
 			return false
 		}
 	} else {
 		md5sum, err := fileutils2.MD5(localPath)
 		if err != nil {
-			log.Errorf("IsActive md5 fail %s", err)
+			log.Errorf("IsActive md5 fail %s for %s", err, localPath)
 			return false
 		}
 		if chksum != md5sum {
-			log.Errorf("IsActive checksum mismatch")
+			log.Errorf("IsActive checksum mismatch: %s", localPath)
 			return false
 		}
 	}

pkg/image/tasks/image_convert_task.go

@@ -25,8 +25,8 @@ func (self *ImageConvertTask) OnInit(ctx context.Context, obj db.IStandaloneMode
 
 	self.SetStage("OnConvertComplete", nil)
 	taskman.LocalTaskRun(self, func() (jsonutils.JSONObject, error) {
-		image.SetStatus(self.UserCred, models.IMAGE_STATUS_CONVERTING, "start convert")
-		defer image.SetStatus(self.UserCred, models.IMAGE_STATUS_ACTIVE, "convert failed")
+		// image.SetStatus(self.UserCred, models.IMAGE_STATUS_CONVERTING, "start convert")
+		// defer image.SetStatus(self.UserCred, models.IMAGE_STATUS_ACTIVE, "convert failed")
 		err := image.ConvertAllSubformats()
 		return nil, err
 	})

pkg/util/qemuimg/qemuimg.go

@@ -23,6 +23,16 @@ var (
 	ErrUnsupportedFormat = errors.New("unsupported format")
 )
 
+type TIONiceLevel int
+
+const (
+	// The scheduling class. 0 for none, 1 for real time, 2 for best-effort, 3 for idle.
+	IONiceNone       = TIONiceLevel(0)
+	IONiceRealTime   = TIONiceLevel(1)
+	IONiceBestEffort = TIONiceLevel(2)
+	IONiceIdle       = TIONiceLevel(3)
+)
+
 type SQemuImage struct {
 	Path            string
 	Password        string
@@ -34,14 +44,23 @@ type SQemuImage struct {
 	Compat          string
 	Encryption      bool
 	Subformat       string
+	IoLevel         TIONiceLevel
 }
 
 func NewQemuImage(path string) (*SQemuImage, error) {
 	return NewEncryptedQemuImage(path, "")
 }
 
+func NewQemuImageWithIOLevel(path string, ioLevel TIONiceLevel) (*SQemuImage, error) {
+	return NewEncryptedQemuImageWithIOLevel(path, "", IONiceNone)
+}
+
 func NewEncryptedQemuImage(path string, password string) (*SQemuImage, error) {
-	qemuImg := SQemuImage{Path: path, Password: password}
+	return NewEncryptedQemuImageWithIOLevel(path, password, IONiceNone)
+}
+
+func NewEncryptedQemuImageWithIOLevel(path string, password string, ioLevel TIONiceLevel) (*SQemuImage, error) {
+	qemuImg := SQemuImage{Path: path, Password: password, IoLevel: ioLevel}
 	err := qemuImg.parse()
 	if err != nil {
 		return nil, err
@@ -154,7 +173,8 @@ func (img *SQemuImage) doConvert(name string, format TImageFormat, options []str
 	if !img.IsValid() {
 		return fmt.Errorf("self is not valid")
 	}
-	cmdline := []string{"convert"}
+	cmdline := []string{"-c", strconv.Itoa(int(img.IoLevel)),
+		qemutils.GetQemuImg(), "convert"}
 	if compact {
 		cmdline = append(cmdline, "-c")
 	}
@@ -170,7 +190,7 @@ func (img *SQemuImage) doConvert(name string, format TImageFormat, options []str
 	}
 	cmdline = append(cmdline, img.Path, name)
 	log.Infof("XXXX qemu-img command: %s", cmdline)
-	cmd := exec.Command(qemutils.GetQemuImg(), cmdline...)
+	cmd := exec.Command("ionice", cmdline...)
 	if len(img.Password) > 0 || len(password) > 0 {
 		input := ""
 		if len(img.Password) > 0 {
@@ -358,7 +378,8 @@ func (img *SQemuImage) create(sizeMB int, format TImageFormat, options []string)
 	if img.IsValid() {
 		return fmt.Errorf("create: the image is valid??? %s", img.Format)
 	}
-	args := []string{"create", "-f", format.String()}
+	args := []string{"-c", strconv.Itoa(int(img.IoLevel)),
+		qemutils.GetQemuImg(), "create", "-f", format.String()}
 	if len(options) > 0 {
 		args = append(args, "-o", strings.Join(options, ","))
 	}
@@ -366,8 +387,7 @@ func (img *SQemuImage) create(sizeMB int, format TImageFormat, options []string)
 	if sizeMB > 0 {
 		args = append(args, fmt.Sprintf("%dM", sizeMB))
 	}
-	log.Debugf("%s %s", qemutils.GetQemuImg(), args)
-	cmd := exec.Command(qemutils.GetQemuImg(), args...)
+	cmd := exec.Command("ionice", args...)
 	err := cmd.Run()
 	if err != nil {
 		log.Errorf("create error %s", err)
@@ -414,7 +434,8 @@ func (img *SQemuImage) Resize(sizeMB int) error {
 	if !img.IsValid() {
 		return fmt.Errorf("self is not valid")
 	}
-	cmd := exec.Command(qemutils.GetQemuImg(), "resize", img.Path, fmt.Sprintf("%dM", sizeMB))
+	cmd := exec.Command("ionice", "-c", strconv.Itoa(int(img.IoLevel)),
+		qemutils.GetQemuImg(), "resize", img.Path, fmt.Sprintf("%dM", sizeMB))
 	err := cmd.Run()
 	if err != nil {
 		log.Errorf("resize fail %s", err)
@@ -427,12 +448,13 @@ func (img *SQemuImage) Rebase(backPath string, force bool) error {
 	if !img.IsValid() {
 		return fmt.Errorf("self is not valid")
 	}
-	args := []string{"rebase"}
+	args := []string{"-c", strconv.Itoa(int(img.IoLevel)),
+		qemutils.GetQemuImg(), "rebase"}
 	if force {
 		args = append(args, "-u")
 	}
 	args = append(args, "-b", backPath, img.Path)
-	cmd := exec.Command(qemutils.GetQemuImg(), args...)
+	cmd := exec.Command("ionice", args...)
 	err := cmd.Run()
 	if err != nil {
 		log.Errorf("rebase fail %s", err)