Merge pull request #13296 from rainzm/automated-cherry-pick-of-#13293-upstream-release-3.7

Automated cherry pick of #13293: fix(glance): convert image in s3 storage

pkg/image/models/image_subs.go

@@ -23,13 +23,13 @@ import (
 	"strings"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/image"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/image/options"
 	"yunion.io/x/onecloud/pkg/image/torrent"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
-	"yunion.io/x/onecloud/pkg/util/qemuimg"
 	"yunion.io/x/onecloud/pkg/util/torrentutils"
 )
 
@@ -118,32 +118,31 @@ func (self *SImageSubformat) DoConvert(image *SImage) error {
 }
 
 func (self *SImageSubformat) Save(image *SImage) error {
+	var err error
+	defer func() {
+		if err != nil {
+			db.Update(self, func() error {
+				self.Status = api.IMAGE_STATUS_SAVE_FAIL
+				return nil
+			})
+		}
+	}()
 	if self.Status == api.IMAGE_STATUS_ACTIVE {
 		return nil
 	}
-	if self.Status != api.IMAGE_STATUS_QUEUED {
-		return nil // httperrors.NewInvalidStatusError("cannot save in status %s", self.Status)
-	}
-	location := image.GetPath(self.Format)
-	_, err := db.Update(self, func() error {
+	_, err = db.Update(self, func() error {
 		self.Status = api.IMAGE_STATUS_SAVING
-		self.Location = fmt.Sprintf("%s%s", LocalFilePrefix, location)
 		return nil
 	})
 	if err != nil {
 		log.Errorf("updateStatus fail %s", err)
 		return err
 	}
-	img, err := image.getQemuImage()
+	info, err := storage.ConvertImage(context.Background(), image, self.Format)
 	if err != nil {
-		log.Errorf("image.getQemuImage fail %s", err)
-		return err
-	}
-	nimg, err := img.Clone(location, qemuimg.String2ImageFormat(self.Format), true)
-	if err != nil {
-		log.Errorf("img.Clone fail %s", err)
-		return err
+		return errors.Wrap(err, "unable to ConvertImage")
 	}
+	location := image.GetPath(self.Format)
 	checksum, err := fileutils2.MD5(location)
 	if err != nil {
 		log.Errorf("fileutils2.Md5 fail %s", err)
@@ -155,10 +154,11 @@ func (self *SImageSubformat) Save(image *SImage) error {
 		return err
 	}
 	_, err = db.Update(self, func() error {
-		self.Location = fmt.Sprintf("%s%s", LocalFilePrefix, location)
+		self.Location = info.Location
 		self.Checksum = checksum
 		self.FastHash = fastHash
-		self.Size = nimg.ActualSizeBytes
+		self.Size = info.SizeBytes
+		self.Status = api.IMAGE_STATUS_ACTIVE
 		return nil
 	})
 	if err != nil {
@@ -172,9 +172,9 @@ func (self *SImageSubformat) SaveTorrent() error {
 	if self.TorrentStatus == api.IMAGE_STATUS_ACTIVE {
 		return nil
 	}
-	if self.TorrentStatus != api.IMAGE_STATUS_QUEUED {
-		return nil // httperrors.NewInvalidStatusError("cannot save torrent in status %s", self.Status)
-	}
+	// if self.TorrentStatus != api.IMAGE_STATUS_QUEUED {
+	// 	return nil // httperrors.NewInvalidStatusError("cannot save torrent in status %s", self.Status)
+	// }
 	imgPath := self.GetLocalLocation()
 	torrentPath := filepath.Join(options.Options.TorrentStoreDir, fmt.Sprintf("%s.torrent", filepath.Base(imgPath)))
 	_, err := db.Update(self, func() error {

pkg/image/models/storage.go

@@ -26,7 +26,9 @@ import (
 	"yunion.io/x/onecloud/pkg/apis/image"
 	"yunion.io/x/onecloud/pkg/image/drivers/s3"
 	"yunion.io/x/onecloud/pkg/image/options"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
+	"yunion.io/x/onecloud/pkg/util/qemuimg"
 )
 
 var local Storage = &LocalStorage{}
@@ -89,6 +91,7 @@ type Storage interface {
 	RemoveImage(context.Context, string) error
 
 	IsCheckStatusEnabled() bool
+	ConvertImage(ctx context.Context, image *SImage, targetFormat string) (*SConverImageInfo, error)
 }
 
 type LocalStorage struct{}
@@ -117,6 +120,22 @@ func (s *LocalStorage) GetImage(ctx context.Context, imagePath string) (int64, i
 	return fstat.Size(), f, nil
 }
 
+func (s *LocalStorage) ConvertImage(ctx context.Context, image *SImage, targetFormat string) (*SConverImageInfo, error) {
+	location := image.GetPath(targetFormat)
+	img, err := image.getQemuImage()
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to image.getQemuImage")
+	}
+	nimg, err := img.Clone(location, qemuimg.String2ImageFormat(targetFormat), true)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to img.Clone")
+	}
+	return &SConverImageInfo{
+		Location:  fmt.Sprintf("%s%s", LocalFilePrefix, location),
+		SizeBytes: nimg.ActualSizeBytes,
+	}, nil
+}
+
 func (s *LocalStorage) IsCheckStatusEnabled() bool {
 	return true
 }
@@ -148,6 +167,52 @@ func (s *S3Storage) CleanTempfile(filePath string) error {
 	return nil
 }
 
+func (s *S3Storage) getTempDir() (string, error) {
+	var dir string
+	if options.Options.FilesystemStoreDatadir != "" {
+		dir = options.Options.FilesystemStoreDatadir + "/image-tmp"
+	} else {
+		dir = "/tmp/image-tmp"
+	}
+	if !fileutils2.Exists(dir) {
+		err := procutils.NewCommand("mkdir", "-p", dir).Run()
+		if err != nil {
+			return "", errors.Wrapf(err, "unable to create dir %s", dir)
+		}
+	}
+	return dir, nil
+}
+
+type SConverImageInfo struct {
+	Location  string
+	SizeBytes int64
+}
+
+func (s *S3Storage) ConvertImage(ctx context.Context, image *SImage, targetFormat string) (*SConverImageInfo, error) {
+	tempDir, err := s.getTempDir()
+	if err != nil {
+		return nil, err
+	}
+	location := fmt.Sprintf("%s/%s.%s", tempDir, image.GetId(), targetFormat)
+	img, err := image.getQemuImage()
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to image.getQemuImage")
+	}
+	nimg, err := img.Clone(location, qemuimg.String2ImageFormat(targetFormat), true)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to img.Clone")
+	}
+	defer s.CleanTempfile(location)
+	s3Location, err := s.SaveImage(ctx, location)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to SaveImage")
+	}
+	return &SConverImageInfo{
+		Location:  s3Location,
+		SizeBytes: nimg.ActualSizeBytes,
+	}, nil
+}
+
 func (s *S3Storage) GetImage(ctx context.Context, imagePath string) (int64, io.ReadCloser, error) {
 	return s3.Get(ctx, imagePathToName(imagePath))
 }