feat(glance): support nfs as backend storage (#25240)

pkg/apis/image/consts.go

@@ -51,9 +51,13 @@ const (
 
 	LocalFilePrefix = "file://"
 	S3Prefix        = "s3://"
+	NfsPrefix       = "nfs://"
+
+	NfsSubDirName = "images"
 
 	IMAGE_STORAGE_DRIVER_LOCAL = "local"
 	IMAGE_STORAGE_DRIVER_S3    = "s3"
+	IMAGE_STORAGE_DRIVER_NFS   = "nfs"
 
 	// image properties
 	IMAGE_OS_ARCH              = "os_arch"

pkg/compute/hostdrivers/esxi.go

@@ -79,7 +79,7 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, userCred
 	hostCacheImage := models.StoragecachedimageManager.GetStoragecachedimage(storageCache.GetId(), cacheImage.GetId())
 	if hostCacheImage == nil {
 		zone, _ := host.GetZone()
-		srcHostCacheImage, err = cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_ESXI, []string{host.Id},
+		srcHostCacheImage, err = cacheImage.ChooseSourceStoragecacheInRange([]string{api.HOST_TYPE_ESXI}, []string{host.Id},
 			[]interface{}{zone, host.GetCloudprovider()})
 		if err != nil {
 			return err

pkg/compute/hostdrivers/kvm.go

@@ -220,7 +220,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, userCred
 	if srcHost != nil {
 		rangeObjs = append(rangeObjs, srcHost)
 	}
-	srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_HYPERVISOR, []string{host.Id}, rangeObjs)
+	srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange([]string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_CONTAINER}, []string{host.Id}, rangeObjs)
 	if err != nil {
 		return errors.Wrapf(err, "Choose source storagecache")
 	}

pkg/compute/models/cachedimages.go

@@ -611,7 +611,7 @@ func (cachedImage *SCachedimage) addRefCount() {
 	}
 }
 
-func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, excludes []string, rangeObjs []interface{}) (*SStoragecachedimage, error) {
+func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType []string, excludes []string, rangeObjs []interface{}) (*SStoragecachedimage, error) {
 	storageCachedImage := StoragecachedimageManager.Query().SubQuery()
 	storage := StorageManager.Query().SubQuery()
 	hostStorage := HoststorageManager.Query().SubQuery()
@@ -635,7 +635,7 @@ func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType string
 		q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes))
 	}
 	if len(hostType) > 0 {
-		q = q.Filter(sqlchemy.Equals(host.Field("host_type"), hostType))
+		q = q.Filter(sqlchemy.In(host.Field("host_type"), hostType))
 	}
 
 	for _, rangeObj := range rangeObjs {

pkg/hostman/storageman/imagecache_local.go

@@ -211,6 +211,9 @@ func (l *SLocalImageCache) prepare(ctx context.Context, input api.CacheImageInpu
 
 	l.remoteFile = remotefile.NewRemoteFile(ctx, url,
 		l.GetPath(), false, input.Checksum, -1, nil, l.GetTmpPath(), input.SrcUrl)
+	if l.Manager.GetStorageType() == api.STORAGE_NFS {
+		l.remoteFile.NfsSetTargetStorageId(l.Manager.GetStorageId(), l.Manager.GetStoragePath())
+	}
 	return false, nil
 }
 

pkg/hostman/storageman/imagecachemanager_base.go

@@ -53,6 +53,8 @@ type IImageCacheManger interface {
 
 	IsLocal() bool
 	GetStorageType() string
+	GetStorageId() string
+	GetStoragePath() string
 
 	// for diskhandler
 	PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error)

pkg/hostman/storageman/imagecachemanager_local.go

@@ -75,6 +75,20 @@ func (c *SLocalImageCacheManager) GetStorageType() string {
 	return c.storage.StorageType()
 }
 
+func (c *SLocalImageCacheManager) GetStorageId() string {
+	if c.storage == nil {
+		return ""
+	}
+	return c.storage.GetId()
+}
+
+func (c *SLocalImageCacheManager) GetStoragePath() string {
+	if c.storage == nil {
+		return ""
+	}
+	return c.storage.GetPath()
+}
+
 func (c *SLocalImageCacheManager) loadCache(ctx context.Context) {
 	if len(c.cachePath) == 0 {
 		return

pkg/hostman/storageman/imagecachemanager_lvm.go

@@ -65,6 +65,14 @@ func (c *SLVMImageCacheManager) GetStorageType() string {
 	return c.storage.StorageType()
 }
 
+func (s *SLVMImageCacheManager) GetStorageId() string {
+	return s.storage.GetId()
+}
+
+func (s *SLVMImageCacheManager) GetStoragePath() string {
+	return s.storage.GetPath()
+}
+
 func (c *SLVMImageCacheManager) Lvmlockd() bool {
 	return c.lvmlockd
 }

pkg/hostman/storageman/imagecachemanager_rbd.go

@@ -118,6 +118,14 @@ func (c *SRbdImageCacheManager) GetStorageType() string {
 	return c.storage.StorageType()
 }
 
+func (c *SRbdImageCacheManager) GetStorageId() string {
+	return c.storage.GetId()
+}
+
+func (c *SRbdImageCacheManager) GetStoragePath() string {
+	return c.storage.GetPath()
+}
+
 func (c *SRbdImageCacheManager) GetPath() string {
 	return c.Pool
 }

pkg/hostman/storageman/remotefile/nfsremotefile.go

@@ -0,0 +1,32 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remotefile
+
+import (
+	"os"
+	"path"
+
+	"yunion.io/x/log"
+)
+
+type NfsRemoteFileInfo struct {
+	NfsImagePath string `json:"nfs_image_path"`
+}
+
+func (i *NfsRemoteFileInfo) nfsLinkImage(targetPath, storagePath string) error {
+	srcPath := path.Join(storagePath, i.NfsImagePath)
+	log.Infof("NfsRemoteFileInfo start link %s to %s", srcPath, targetPath)
+	return os.Link(srcPath, targetPath)
+}

pkg/hostman/storageman/remotefile/remotefile.go

@@ -48,27 +48,30 @@ type SImageDesc struct {
 }
 
 type SRemoteFile struct {
-	ctx          context.Context
-	url          string
-	downloadUrl  string
-	localPath    string
-	tmpPath      string
-	preChksum    string
-	compress     bool
-	timeout      time.Duration
-	extraHeaders map[string]string
+	ctx                  context.Context
+	url                  string
+	downloadUrl          string
+	localPath            string
+	tmpPath              string
+	preChksum            string
+	compress             bool
+	timeout              time.Duration
+	extraHeaders         map[string]string
+	nfsTargetStorageId   string
+	nfsTargetStoragePath string
 
 	chksum string
 	format string
 	name   string
 
-	s3Info *S3RemoteFileInfo
+	s3Info  *S3RemoteFileInfo
+	nfsInfo *NfsRemoteFileInfo
 }
 
 func NewRemoteFile(
 	ctx context.Context, url, localPath string, compress bool,
 	PreChksum string, timeout int, extraHeaders map[string]string,
-	tmpPath string, downloadUrl string,
+	tmpPath, downloadUrl string,
 ) *SRemoteFile {
 	if timeout <= 0 {
 		timeout = 24 * 3600 //24 hours
@@ -90,6 +93,11 @@ func NewRemoteFile(
 	}
 }
 
+func (r *SRemoteFile) NfsSetTargetStorageId(storageId, storagePath string) {
+	r.nfsTargetStorageId = storageId
+	r.nfsTargetStoragePath = storagePath
+}
+
 func (r *SRemoteFile) GetFormat() string {
 	return r.format
 }
@@ -230,8 +238,12 @@ func (r *SRemoteFile) downloadS3(callback func(progress, progressMbps float64, t
 }
 
 func (r *SRemoteFile) downloadInternal(getData bool, preChksum string, callback func(progress, progressMbps float64, totalSizeMb int64)) error {
-	if getData && r.s3Info != nil {
-		return r.downloadS3(callback)
+	if getData {
+		if r.s3Info != nil {
+			return r.downloadS3(callback)
+		} else if r.nfsInfo != nil {
+			return r.nfsInfo.nfsLinkImage(r.tmpPath, r.nfsTargetStoragePath)
+		}
 	}
 
 	var header = http.Header{}
@@ -363,4 +375,12 @@ func (r *SRemoteFile) setProperties(header http.Header) {
 			r.s3Info.Bucket = s3Bucket
 		}
 	}
+	if nfsStorageId := header.Get("X-Image-Meta-Nfs_storage_id"); len(nfsStorageId) > 0 && r.nfsTargetStorageId == nfsStorageId {
+		if imgPath := header.Get("X-Image-Meta-Nfs_image_path"); len(imgPath) > 0 {
+			r.nfsInfo = &NfsRemoteFileInfo{
+				NfsImagePath: imgPath,
+			}
+
+		}
+	}
 }

pkg/image/models/images.go

@@ -400,6 +400,11 @@ func (img *SImage) GetExtraDetailsHeaders(ctx context.Context, userCred mcclient
 		headers[fmt.Sprintf("%s%s", modules.IMAGE_META, "s3_info_sign_ver")] = options.Options.S3SignVersion
 	}
 
+	if strings.HasPrefix(img.Location, api.NfsPrefix) {
+		headers[fmt.Sprintf("%s%s", modules.IMAGE_META, "nfs_storage_id")] = options.Options.NfsStorageId
+		headers[fmt.Sprintf("%s%s", modules.IMAGE_META, "nfs_image_path")] = img.Location[len(api.NfsPrefix):]
+	}
+
 	return headers
 }
 
@@ -461,6 +466,8 @@ func (self *SImage) GetPath(format string) string {
 	path := filepath.Join(options.Options.FilesystemStoreDatadir, self.Id)
 	if options.Options.StorageDriver == api.IMAGE_STORAGE_DRIVER_S3 {
 		path = filepath.Join(options.Options.S3MountPoint, self.Id)
+	} else if options.Options.StorageDriver == api.IMAGE_STORAGE_DRIVER_NFS {
+		path = filepath.Join(options.Options.NfsMountPoint, api.NfsSubDirName, self.Id)
 	}
 	if len(format) > 0 {
 		path = fmt.Sprintf("%s.%s", path, format)
@@ -1180,6 +1187,8 @@ func (self *SImage) GetLocalLocation() string {
 		return self.Location[len(api.LocalFilePrefix):]
 	} else if strings.HasPrefix(self.Location, api.S3Prefix) {
 		return path.Join(options.Options.S3MountPoint, self.Location[len(api.S3Prefix):])
+	} else if strings.HasPrefix(self.Location, api.NfsPrefix) {
+		return path.Join(options.Options.NfsMountPoint, self.Location[len(api.NfsPrefix):])
 	} else {
 		return ""
 	}
@@ -1190,6 +1199,8 @@ func (self *SImage) GetPrefix() string {
 		return api.LocalFilePrefix
 	} else if strings.HasPrefix(self.Location, api.S3Prefix) {
 		return api.S3Prefix
+	} else if strings.HasPrefix(self.Location, api.NfsPrefix) {
+		return api.NfsPrefix
 	} else {
 		return api.LocalFilePrefix
 	}
@@ -1198,6 +1209,8 @@ func (self *SImage) GetPrefix() string {
 func (self *SImage) GetNewLocation(newLocalPath string) string {
 	if strings.HasPrefix(self.Location, api.S3Prefix) {
 		return api.S3Prefix + path.Base(newLocalPath)
+	} else if strings.HasPrefix(self.Location, api.NfsPrefix) {
+		return api.NfsPrefix + path.Join(api.NfsSubDirName, path.Base(newLocalPath))
 	} else {
 		return api.LocalFilePrefix + newLocalPath
 	}

pkg/image/models/storage.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"strings"
 
 	"yunion.io/x/pkg/errors"
@@ -33,6 +34,7 @@ import (
 
 var local IImageStorage = &LocalStorage{}
 var s3Instance IImageStorage = &S3Storage{}
+var nfsInstance IImageStorage = &NFSStorage{}
 var storage IImageStorage
 
 func GetStorage() IImageStorage {
@@ -43,6 +45,8 @@ func GetImage(ctx context.Context, location string) (int64, io.ReadCloser, error
 	switch {
 	case strings.HasPrefix(location, image.S3Prefix):
 		return s3Instance.GetImage(ctx, location[len(image.S3Prefix):])
+	case strings.HasPrefix(location, image.NfsPrefix):
+		return nfsInstance.GetImage(ctx, location[len(image.NfsPrefix):])
 	case strings.HasPrefix(location, image.LocalFilePrefix):
 		return local.GetImage(ctx, location[len(image.LocalFilePrefix):])
 	default:
@@ -54,6 +58,8 @@ func RemoveImage(ctx context.Context, location string) error {
 	switch {
 	case strings.HasPrefix(location, image.S3Prefix):
 		return s3Instance.RemoveImage(ctx, location[len(image.S3Prefix):])
+	case strings.HasPrefix(location, image.NfsPrefix):
+		return nfsInstance.RemoveImage(ctx, location[len(image.NfsPrefix):])
 	case strings.HasPrefix(location, image.LocalFilePrefix):
 		return local.RemoveImage(ctx, location[len(image.LocalFilePrefix):])
 	default:
@@ -65,6 +71,8 @@ func IsCheckStatusEnabled(img *SImage) bool {
 	switch {
 	case strings.HasPrefix(img.Location, image.S3Prefix):
 		return s3Instance.IsCheckStatusEnabled()
+	case strings.HasPrefix(img.Location, image.NfsPrefix):
+		return nfsInstance.IsCheckStatusEnabled()
 	case strings.HasPrefix(img.Location, image.LocalFilePrefix):
 		return local.IsCheckStatusEnabled()
 	default:
@@ -76,6 +84,8 @@ func Init(storageBackend string) {
 	switch storageBackend {
 	case image.IMAGE_STORAGE_DRIVER_LOCAL:
 		storage = &LocalStorage{}
+	case image.IMAGE_STORAGE_DRIVER_NFS:
+		storage = &NFSStorage{}
 	case image.IMAGE_STORAGE_DRIVER_S3:
 		storage = &S3Storage{}
 	default:
@@ -144,6 +154,41 @@ func (s *LocalStorage) RemoveImage(ctx context.Context, imagePath string) error
 	return os.Remove(imagePath)
 }
 
+type NFSStorage struct {
+	LocalStorage
+}
+
+func (s *NFSStorage) Type() string {
+	return image.IMAGE_STORAGE_DRIVER_NFS
+}
+
+func (s *NFSStorage) SaveImage(ctx context.Context, imagePath string, progresser func(saved int64)) (string, error) {
+	imageName := imagePathToName(imagePath)
+	imageNewpath := path.Join(options.Options.NfsMountPoint, image.NfsSubDirName, imageName)
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("cp", imagePath, imageNewpath).Output()
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to copy %s to %s: %s", imagePath, imageNewpath, out)
+	}
+
+	return fmt.Sprintf("%s%s", image.NfsPrefix, path.Join(image.NfsSubDirName, imageName)), nil
+}
+
+func (s *NFSStorage) ConvertImage(ctx context.Context, oimg *SImage, targetFormat string, progresser func(saved int64)) (*SConverImageInfo, error) {
+	location := oimg.GetPath(targetFormat)
+	img, err := oimg.getQemuImage()
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to image.getQemuImage")
+	}
+	nimg, err := img.Clone(location, qemuimgfmt.String2ImageFormat(targetFormat), true)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to img.Clone")
+	}
+	return &SConverImageInfo{
+		Location:  fmt.Sprintf("%s%s", image.NfsPrefix, location),
+		SizeBytes: nimg.ActualSizeBytes,
+	}, nil
+}
+
 type S3Storage struct{}
 
 func imagePathToName(imagePath string) string {

pkg/image/options/options.go

@@ -48,7 +48,7 @@ type SImageOptions struct {
 
 	// DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"`
 
-	StorageDriver string `help:"image backend storage" default:"local" choices:"s3|local"`
+	StorageDriver string `help:"image backend storage" default:"local" choices:"s3|local|nfs"`
 
 	S3MountPoint       string `help:"s3fs mount point" default:"/opt/cloud/workspace/data/glance/s3images"`
 	S3CheckImageStatus bool   `help:"Enable s3 check image status"`
@@ -58,6 +58,10 @@ type SImageOptions struct {
 
 	S3DirectDownload bool `help:"enable s3 direct download" default:"false"`
 
+	NfsStorageId    string `help:"region nfs storage id or name used as glance filesystem backend"`
+	NfsMountOptions string `help:"nfs mount options for glance filesystem backend"`
+	NfsMountPoint   string `help:"nfs mount point" default:"/opt/cloud/workspace/data/glance/nfsimages"`
+
 	ImageStreamWorkerCount int `help:"Image stream worker count" default:"10"`
 
 	VerifyImageStatusIntervalMinutes int `help:"verify image status periodically, default 15 minutes" default:"15"`

pkg/image/service/nfs.go

@@ -0,0 +1,112 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	imageapi "yunion.io/x/onecloud/pkg/apis/image"
+	"yunion.io/x/onecloud/pkg/image/options"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+func initNFS() error {
+	if options.Options.StorageDriver != imageapi.IMAGE_STORAGE_DRIVER_NFS {
+		return nil
+	}
+	if len(options.Options.NfsStorageId) == 0 {
+		return fmt.Errorf("nfs_storage_id is required when storage_driver is nfs")
+	}
+
+	storage, err := getNFSStorage(options.Options.NfsStorageId)
+	if err != nil {
+		return errors.Wrapf(err, "get nfs storage %s", options.Options.NfsStorageId)
+	}
+	if storage.StorageType != computeapi.STORAGE_NFS {
+		return fmt.Errorf("storage %s is %s, not nfs", options.Options.NfsStorageId, storage.StorageType)
+	}
+	if storage.StorageConf == nil {
+		return fmt.Errorf("storage %s has empty storage_conf", options.Options.NfsStorageId)
+	}
+
+	host, err := storage.StorageConf.GetString("nfs_host")
+	if err != nil {
+		return errors.Wrapf(err, "storage %s missing nfs_host", options.Options.NfsStorageId)
+	}
+	sharedDir, err := storage.StorageConf.GetString("nfs_shared_dir")
+	if err != nil {
+		return errors.Wrapf(err, "storage %s missing nfs_shared_dir", options.Options.NfsStorageId)
+	}
+
+	return mountNFS(host, sharedDir, options.Options.NfsMountPoint, options.Options.NfsMountOptions)
+}
+
+func getNFSStorage(storageId string) (*computeapi.StorageDetails, error) {
+	params := jsonutils.NewDict()
+	params.Set("details", jsonutils.JSONTrue)
+	obj, err := compute.Storages.Get(auth.GetAdminSession(context.Background(), options.Options.Region), storageId, params)
+	if err != nil {
+		return nil, err
+	}
+	storage := new(computeapi.StorageDetails)
+	if err := obj.Unmarshal(storage); err != nil {
+		return nil, errors.Wrap(err, "unmarshal storage details")
+	}
+	return storage, nil
+}
+
+func mountNFS(host, sharedDir, mountPoint, mountOptions string) error {
+	if err := os.MkdirAll(mountPoint, 0755); err != nil {
+		return errors.Wrapf(err, "mkdir %s", mountPoint)
+	}
+	if err := os.MkdirAll(path.Join(mountPoint, imageapi.NfsSubDirName), 0755); err != nil {
+		return errors.Wrapf(err, "mkdir %s", mountPoint)
+	}
+	source := fmt.Sprintf("%s:%s", host, sharedDir)
+	if out, err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", mountPoint).Output(); err == nil {
+		log.Infof("%s has already been mounted as glance nfs store", mountPoint)
+		return nil
+	} else {
+		log.Infof("%s is not mounted yet: %s", mountPoint, strings.TrimSpace(string(out)))
+	}
+
+	args := []string{"-t", "nfs"}
+	if len(mountOptions) > 0 {
+		args = append(args, "-o", mountOptions)
+	}
+	args = append(args, source, mountPoint)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	out, err := procutils.NewRemoteCommandContextAsFarAsPossible(ctx, "mount", args...).Output()
+	if err != nil {
+		return errors.Wrapf(err, "mount %s to %s failed: %s", source, mountPoint, out)
+	}
+	log.Infof("mounted nfs %s to glance filesystem store %s", source, mountPoint)
+
+	return nil
+}

pkg/image/service/service.go

@@ -105,10 +105,6 @@ func StartService() {
 		procutils.SetRemoteExecutor()
 	}
 
-	if !opts.IsSlaveNode {
-
-	}
-
 	trackers := torrent.GetTrackers()
 	if len(trackers) == 0 {
 		log.Errorf("no valid torrent-tracker")
@@ -177,7 +173,11 @@ func startMasterTasks(app *appsrv.Application, opts *options.SImageOptions) {
 		} else {
 			log.Infof("storage driver is not s3 and no valid s3 options, skip init s3 client")
 		}
-		// check image after s3 mounted
+
+		if err := initNFS(); err != nil {
+			log.Fatalf("fail to init nfs storage: %s", err)
+		}
+		// check image after storage mounted
 		models.CheckImages(app.GetContext())
 	}()