gpfs

- support gpfs
- fix cache non lvm

pkg/hostman/guestman/guestman.go

@@ -40,7 +40,6 @@ import (
 	"yunion.io/x/onecloud/pkg/util/cgrouputils"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/netutils2"
-	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/seclib2"
 	"yunion.io/x/onecloud/pkg/util/timeutils2"
 )
@@ -482,61 +481,14 @@ func (m *SGuestManager) DestPrepareMigrate(ctx context.Context, params interface
 			return nil, fmt.Errorf("Target storage %s not found", migParams.TargetStorageId)
 		}
 
-		disks, _ := migParams.Desc.GetArray("disks")
-		for i, diskinfo := range disks {
-			var (
-				diskId, _    = diskinfo.GetString("disk_id")
-				snapshots, _ = migParams.SrcSnapshots.GetArray(diskId)
-				disk         = iStorage.CreateDisk(diskId)
-			)
-
-			if disk == nil {
-				return nil, fmt.Errorf(
-					"Storage %s create disk %s failed", iStorage.GetId(), diskId)
-			}
-
-			// prepare disk snapshot dir
-			if len(snapshots) > 0 && !fileutils2.Exists(disk.GetSnapshotDir()) {
-				_, err := procutils.NewCommand("mkdir", "-p", disk.GetSnapshotDir()).Run()
-				if err != nil {
-					return nil, err
-				}
-			}
-
-			// create snapshots form remote url
-			diskStorageId, _ := diskinfo.GetString("storage_id")
-			for _, snapshotId := range snapshots {
-				snapId, _ := snapshotId.GetString()
-
-				snapshotUrl := fmt.Sprintf("%s/%s/%s/%s",
-					migParams.SnapshotsUri, diskStorageId, diskId, snapId)
-				snapshotPath := path.Join(disk.GetSnapshotDir(), snapId)
-				log.Infof("Disk %s snapshot %s url: %s", diskId, snapId, snapshotUrl)
-				iStorage.CreateSnapshotFormUrl(ctx, snapshotUrl, diskId, snapshotPath)
-			}
-
-			if migParams.LiveMigrate {
-				// create local disk
-				backingFile, _ := migParams.DisksBackingFile.GetString(diskId)
-				size, _ := diskinfo.Int("size")
-				_, err := disk.CreateRaw(ctx, int(size), "qcow2", "", false, "", backingFile)
-				if err != nil {
-					log.Errorln(err)
-					return nil, err
-				}
-			} else {
-				// download disk form remote url
-				diskUrl := fmt.Sprintf("%s/%s/%s", migParams.DisksUri, diskStorageId, diskId)
-				if err := disk.CreateFromUrl(ctx, diskUrl); err != nil {
-					log.Errorln(err)
-					return nil, err
-				}
-			}
-			diskDesc, _ := disks[i].(*jsonutils.JSONDict)
-			diskDesc.Set("path", jsonutils.NewString(disk.GetPath()))
+		err := iStorage.DestinationPrepareMigrate(
+			ctx, migParams.LiveMigrate, migParams.DisksUri, migParams.SnapshotsUri,
+			migParams.Desc, migParams.DisksBackingFile, migParams.SrcSnapshots)
+		if err != nil {
+			return nil, fmt.Errorf("dest prepare migrate failed %s", err)
 		}
 
-		if err := guest.SaveDesc(migParams.Desc); err != nil {
+		if err = guest.SaveDesc(migParams.Desc); err != nil {
 			log.Errorln(err)
 			return nil, err
 		}

pkg/hostman/storageman/core.go

@@ -24,6 +24,7 @@ import (
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
@@ -44,8 +45,8 @@ type SStorageManager struct {
 	LocalStorageImagecacheManager IImageCacheManger
 	// AgentStorageImagecacheManager IImageCacheManger
 
-	RbdStorageImagecacheManagers map[string]IImageCacheManger
-	NfsStorageImagecacheManagers map[string]IImageCacheManger
+	RbdStorageImagecacheManagers        map[string]IImageCacheManger
+	SharedFileStorageImagecacheManagers map[string]IImageCacheManger
 }
 
 func NewStorageManager(host hostutils.IHost) (*SStorageManager, error) {
@@ -87,8 +88,8 @@ func NewStorageManager(host hostutils.IHost) (*SStorageManager, error) {
 }
 
 func (s *SStorageManager) Remove(storage IStorage) {
-	if storage.StorageType() == api.STORAGE_NFS {
-		delete(s.NfsStorageImagecacheManagers, storage.GetStoragecacheId())
+	if utils.IsInStringArray(storage.StorageType(), api.SHARED_FILE_STORAGE) {
+		delete(s.SharedFileStorageImagecacheManagers, storage.GetStoragecacheId())
 	} else if storage.StorageType() == api.STORAGE_RBD {
 		delete(s.RbdStorageImagecacheManagers, storage.GetStoragecacheId())
 	}
@@ -167,40 +168,6 @@ func (s *SStorageManager) initLocalStorageImagecache() error {
 	}
 }
 
-// func (s *SStorageManager) initAgentStorageImagecache() {
-// 	s.AgentStorageImagecacheManager = NewAgentImageCacheManager(s)
-// }
-
-// func (s *SStorageManager) initAgentStorage() error {
-// 	var cacheDir = "agent_tmp"
-// 	var spath = options.HostOptions.AgentTempPath
-// 	var limit = options.HostOptions.AgentTempLimit
-// 	if len(spath) == 0 {
-// 		var err error
-// 		spath, err = s.getLeasedUsedLocalStorage(cacheDir, limit)
-// 		if err != nil {
-// 			return err
-// 		}
-// 	}
-// 	if len(spath) != nil {
-// 		// TODO: NewAgentStorage
-// 		s.AgentStorage = NewAgentStorage(s, spath)
-// 	} else {
-// 		return fmt.Errorf("Cannot allocate agent storage")
-// 	}
-// }
-
-// func (s *SStorageManager) AddNfsStorage(storagecacheId, cachePath string) {
-// 	if len(cachePath) == 0 {
-// 		return
-// 	}
-// 	if s.NfsStorageImagecacheManagers == nil {
-// 		s.NfsStorageImagecacheManagers = make(map[string]IImageCacheManger, 0)
-// 	}
-// 	s.NfsStorageImagecacheManagers[storagecacheId] = NewLocalImageCacheManager(s, cachePath,
-// 		options.HostOptions.ImageCacheLimit, true, storagecacheId)
-// }
-
 func (s *SStorageManager) GetStorage(storageId string) IStorage {
 	for _, storage := range s.Storages {
 		if storage.GetId() == storageId {
@@ -253,7 +220,7 @@ func (s *SStorageManager) GetStoragecacheById(scId string) IImageCacheManger {
 	if s.LocalStorageImagecacheManager.GetId() == scId {
 		return s.LocalStorageImagecacheManager
 	}
-	if sc, ok := s.NfsStorageImagecacheManagers[scId]; ok {
+	if sc, ok := s.SharedFileStorageImagecacheManagers[scId]; ok {
 		return sc
 	}
 	if sc, ok := s.RbdStorageImagecacheManagers[scId]; ok {
@@ -267,25 +234,24 @@ func (s *SStorageManager) NewSharedStorageInstance(mountPoint, storageType strin
 }
 
 func (s *SStorageManager) InitSharedStorageImageCache(storageType, storagecacheId, imagecachePath string, storage IStorage) {
-	if storageType == api.STORAGE_NFS {
-		s.InitNfsStorageImagecache(storagecacheId, imagecachePath)
+	if utils.IsInStringArray(storageType, api.SHARED_FILE_STORAGE) {
+		s.InitSharedFileStorageImagecache(storagecacheId, imagecachePath)
 	} else if storageType == api.STORAGE_RBD {
 		if rbdStorage := s.GetStoragecacheById(storagecacheId); rbdStorage == nil {
-			// Done
 			s.AddRbdStorageImagecache(imagecachePath, storage, storagecacheId)
 		}
 	}
 }
 
-func (s *SStorageManager) InitNfsStorageImagecache(storagecacheId, path string) {
+func (s *SStorageManager) InitSharedFileStorageImagecache(storagecacheId, path string) {
 	if len(path) == 0 {
 		return
 	}
-	if s.NfsStorageImagecacheManagers == nil {
-		s.NfsStorageImagecacheManagers = map[string]IImageCacheManger{}
+	if s.SharedFileStorageImagecacheManagers == nil {
+		s.SharedFileStorageImagecacheManagers = map[string]IImageCacheManger{}
 	}
-	if _, ok := s.NfsStorageImagecacheManagers[storagecacheId]; !ok {
-		s.NfsStorageImagecacheManagers[storagecacheId] = NewLocalImageCacheManager(s, path, options.HostOptions.ImageCacheLimit, true, storagecacheId)
+	if _, ok := s.SharedFileStorageImagecacheManagers[storagecacheId]; !ok {
+		s.SharedFileStorageImagecacheManagers[storagecacheId] = NewLocalImageCacheManager(s, path, options.HostOptions.ImageCacheLimit, true, storagecacheId)
 	}
 }
 

pkg/hostman/storageman/disk_nas.go

@@ -1,17 +1,3 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package storageman
 
 import (
@@ -19,25 +5,18 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 )
 
-type SNFSDisk struct {
+type SNasDisk struct {
 	SLocalDisk
 }
 
-func NewNFSDisk(storage IStorage, id string) *SNFSDisk {
-	return &SNFSDisk{
-		SLocalDisk: *NewLocalDisk(storage, id),
-	}
+func NewNasDisk(storage IStorage, id string) *SNasDisk {
+	return &SNasDisk{*NewLocalDisk(storage, id)}
 }
 
-func (d *SNFSDisk) GetType() string {
-	return api.STORAGE_NFS
-}
-
-func (d *SNFSDisk) CreateFromTemplate(ctx context.Context, imageId, format string, size int64) (jsonutils.JSONObject, error) {
+func (d *SNasDisk) CreateFromTemplate(ctx context.Context, imageId, format string, size int64) (jsonutils.JSONObject, error) {
 	imageCacheManager := storageManager.GetStoragecacheById(d.Storage.GetStoragecacheId())
 	ret, err := d.SLocalDisk.createFromTemplate(ctx, imageId, format, imageCacheManager)
 	if err != nil {
@@ -52,3 +31,31 @@ func (d *SNFSDisk) CreateFromTemplate(ctx context.Context, imageId, format strin
 	}
 	return ret, nil
 }
+
+type SNFSDisk struct {
+	SNasDisk
+}
+
+func NewNFSDisk(storage IStorage, id string) *SNFSDisk {
+	return &SNFSDisk{
+		SNasDisk: *NewNasDisk(storage, id),
+	}
+}
+
+func (d *SNFSDisk) GetType() string {
+	return api.STORAGE_NFS
+}
+
+type SGPFSDisk struct {
+	SNasDisk
+}
+
+func NewGPFSDisk(storage IStorage, id string) *SGPFSDisk {
+	return &SGPFSDisk{
+		SNasDisk: *NewNasDisk(storage, id),
+	}
+}
+
+func (d *SGPFSDisk) GetType() string {
+	return api.STORAGE_GPFS
+}

pkg/hostman/storageman/lvmutils.go

@@ -51,9 +51,9 @@ func NewLVMImageConnectUniqueToolSet() *SLVMImageConnectUniqueToolSet {
 
 func (s *SLVMImageConnectUniqueToolSet) CacheNonLvmImagePath(imagePath string) {
 	s.lock.Lock()
-	defer s.lock.Unlock()
-
 	s.nonLvms[imagePath] = struct{}{}
+	s.lock.Unlock()
+	s.Release(imagePath)
 }
 
 func (s *SLVMImageConnectUniqueToolSet) GetPathType(imagePath string) int {

pkg/hostman/storageman/storage_base.go

@@ -27,7 +27,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 )
 
-var (
+const (
 	_RECYCLE_BIN_     = "recycle_bin"
 	_IMGSAVE_BACKUPS_ = "imgsave_backups"
 )
@@ -41,6 +41,7 @@ var (
 	storagesFactories = make([]IStorageFactory, 0)
 )
 
+// for shared storages
 func registerStorageFactory(factory IStorageFactory) {
 	storagesFactories = append(storagesFactories, factory)
 }
@@ -94,6 +95,8 @@ type IStorage interface {
 	GetFuseTmpPath() string
 	GetFuseMountPath() string
 	GetImgsaveBackupPath() string
+
+	DestinationPrepareMigrate(ctx context.Context, liveMigrate bool, disksUri string, snapshotsUri string, desc, disksBackingFile, srcSnapshots jsonutils.JSONObject) error
 }
 
 type SBaseStorage struct {
@@ -296,6 +299,6 @@ func (s *SBaseStorage) CreateDiskFromSnpashot(ctx context.Context, disk IDisk, c
 	return disk.GetDiskDesc(), nil
 }
 
-func (s *SLocalStorage) GetImgsaveBackupPath() string {
-	return s.getSubdirPath(_IMGSAVE_BACKUPS_)
+func (s *SBaseStorage) DestinationPrepareMigrate(ctx context.Context, liveMigrate bool, disksUri string, snapshotsUri string, desc, disksBackingFile, srcSnapshots jsonutils.JSONObject) error {
+	return nil
 }

pkg/hostman/storageman/storage_gpfs.go

@@ -0,0 +1,38 @@
+package storageman
+
+import (
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+)
+
+func init() {
+	registerStorageFactory(&SGPFSStorageFactory{})
+}
+
+type SGPFSStorageFactory struct {
+}
+
+func (factory *SGPFSStorageFactory) NewStorage(manager *SStorageManager, mountPoint string) IStorage {
+	return NewNFSStorage(manager, mountPoint)
+}
+
+func (factory *SGPFSStorageFactory) StorageType() string {
+	return api.STORAGE_GPFS
+}
+
+type SGPFSStorage struct {
+	SNasStorage
+}
+
+func (s *SGPFSStorage) newDisk(diskId string) IDisk {
+	return NewGPFSDisk(s, diskId)
+}
+
+func (s *SGPFSStorage) StorageType() string {
+	return api.STORAGE_GPFS
+}
+
+func NewGPFSStorage(manager *SStorageManager, path string) *SGPFSStorage {
+	ret := &SGPFSStorage{}
+	ret.SNasStorage = *NewNasStorage(manager, path, ret)
+	return ret
+}

pkg/hostman/storageman/storage_local.go

@@ -186,6 +186,10 @@ func (s *SLocalStorage) getSubdirPath(subdir string) string {
 	return path.Join(spath, today)
 }
 
+func (s *SLocalStorage) GetImgsaveBackupPath() string {
+	return s.getSubdirPath(_IMGSAVE_BACKUPS_)
+}
+
 func (s *SLocalStorage) SaveToGlance(ctx context.Context, params interface{}) (jsonutils.JSONObject, error) {
 	data, ok := params.(*jsonutils.JSONDict)
 	if !ok {
@@ -479,4 +483,59 @@ func (s *SLocalStorage) requestDeleteSnapshot(
 	}
 }
 
-/*******************************  END  *****************************/
+func (s *SLocalStorage) DestinationPrepareMigrate(ctx context.Context, liveMigrate bool, disksUri string, snapshotsUri string, desc, disksBackingFile, srcSnapshots jsonutils.JSONObject) error {
+	disks, _ := desc.GetArray("disks")
+	for i, diskinfo := range disks {
+		var (
+			diskId, _    = diskinfo.GetString("disk_id")
+			snapshots, _ = srcSnapshots.GetArray(diskId)
+			disk         = s.CreateDisk(diskId)
+		)
+
+		if disk == nil {
+			return fmt.Errorf(
+				"Storage %s create disk %s failed", s.GetId(), diskId)
+		}
+
+		// prepare disk snapshot dir
+		if len(snapshots) > 0 && !fileutils2.Exists(disk.GetSnapshotDir()) {
+			_, err := procutils.NewCommand("mkdir", "-p", disk.GetSnapshotDir()).Run()
+			if err != nil {
+				return err
+			}
+		}
+
+		// create snapshots form remote url
+		diskStorageId, _ := diskinfo.GetString("storage_id")
+		for _, snapshotId := range snapshots {
+			snapId, _ := snapshotId.GetString()
+
+			snapshotUrl := fmt.Sprintf("%s/%s/%s/%s",
+				snapshotsUri, diskStorageId, diskId, snapId)
+			snapshotPath := path.Join(disk.GetSnapshotDir(), snapId)
+			log.Infof("Disk %s snapshot %s url: %s", diskId, snapId, snapshotUrl)
+			s.CreateSnapshotFormUrl(ctx, snapshotUrl, diskId, snapshotPath)
+		}
+
+		if liveMigrate {
+			// create local disk
+			backingFile, _ := disksBackingFile.GetString(diskId)
+			size, _ := diskinfo.Int("size")
+			_, err := disk.CreateRaw(ctx, int(size), "qcow2", "", false, "", backingFile)
+			if err != nil {
+				log.Errorln(err)
+				return err
+			}
+		} else {
+			// download disk form remote url
+			diskUrl := fmt.Sprintf("%s/%s/%s", disksUri, diskStorageId, diskId)
+			if err := disk.CreateFromUrl(ctx, diskUrl); err != nil {
+				log.Errorln(err)
+				return err
+			}
+		}
+		diskDesc, _ := disks[i].(*jsonutils.JSONDict)
+		diskDesc.Set("path", jsonutils.NewString(disk.GetPath()))
+	}
+	return nil
+}

pkg/hostman/storageman/storage_nas.go

@@ -0,0 +1,73 @@
+package storageman
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/hostman/hostutils"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type INasStorage interface {
+	newDisk(diskId string) IDisk
+}
+
+type SNasStorage struct {
+	SLocalStorage
+
+	ins INasStorage
+}
+
+func NewNasStorage(manager *SStorageManager, path string, ins INasStorage) *SNasStorage {
+	return &SNasStorage{*NewLocalStorage(manager, path, 0), ins}
+}
+
+func (s *SNasStorage) CreateDisk(diskId string) IDisk {
+	s.DiskLock.Lock()
+	defer s.DiskLock.Unlock()
+	disk := s.ins.newDisk(diskId)
+	s.Disks = append(s.Disks, disk)
+	return disk
+}
+
+func (s *SNasStorage) GetDiskById(diskId string) IDisk {
+	s.DiskLock.Lock()
+	defer s.DiskLock.Unlock()
+	for i := 0; i < len(s.Disks); i++ {
+		if s.Disks[i].GetId() == diskId {
+			if s.Disks[i].Probe() == nil {
+				return s.Disks[i]
+			} else {
+				return nil
+			}
+		}
+	}
+	var disk = s.ins.newDisk(diskId)
+	if disk.Probe() == nil {
+		s.Disks = append(s.Disks, disk)
+		return disk
+	} else {
+		return nil
+	}
+}
+
+func (s *SNasStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
+	if len(s.StorageId) == 0 {
+		return nil, fmt.Errorf("Sync nfs storage without storage id")
+	}
+	content := jsonutils.NewDict()
+	content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb())))
+	content.Set("storage_type", jsonutils.NewString(s.StorageType()))
+	content.Set("zone", jsonutils.NewString(s.GetZone()))
+	log.Infof("Sync storage info %s", s.StorageId)
+	res, err := modules.Storages.Put(
+		hostutils.GetComputeSession(context.Background()),
+		s.StorageId, content)
+	if err != nil {
+		log.Errorf("SyncStorageInfo Failed: %s: %s", content, err)
+	}
+	return res, err
+}

pkg/hostman/storageman/storage_nfs.go

@@ -57,44 +57,18 @@ func NewNFSStorage(manager *SStorageManager, path string) *SNFSStorage {
 	return ret
 }
 
+func (s *SNFSStorage) newDisk(diskId string) IDisk {
+	return NewNFSDisk(s, diskId)
+}
+
 func (s *SNFSStorage) StorageType() string {
 	return api.STORAGE_NFS
 }
 
-func (s *SNFSStorage) CreateDisk(diskId string) IDisk {
-	s.DiskLock.Lock()
-	defer s.DiskLock.Unlock()
-	disk := NewNFSDisk(s, diskId)
-	s.Disks = append(s.Disks, disk)
-	return disk
-}
-
-func (s *SNFSStorage) GetDiskById(diskId string) IDisk {
-	s.DiskLock.Lock()
-	defer s.DiskLock.Unlock()
-	for i := 0; i < len(s.Disks); i++ {
-		if s.Disks[i].GetId() == diskId {
-			if s.Disks[i].Probe() == nil {
-				return s.Disks[i]
-			} else {
-				return nil
-			}
-		}
-	}
-	var disk = NewNFSDisk(s, diskId)
-	if disk.Probe() == nil {
-		s.Disks = append(s.Disks, disk)
-		return disk
-	} else {
-		return nil
-	}
-}
-
 func (s *SNFSStorage) SyncStorageInfo() (jsonutils.JSONObject, error) {
 	if len(s.StorageId) == 0 {
 		return nil, fmt.Errorf("Sync nfs storage without storage id")
 	}
-
 	content := jsonutils.NewDict()
 	content.Set("capacity", jsonutils.NewInt(int64(s.GetAvailSizeMb())))
 	content.Set("storage_type", jsonutils.NewString(s.StorageType()))
@@ -130,7 +104,7 @@ func (s *SNFSStorage) checkAndMount() error {
 	}
 	sharedDir, err := s.StorageConf.GetString("nfs_shared_dir")
 	if err != nil {
-		return fmt.Errorf("Storage conf missing nfs_shared_dir ")
+		return fmt.Errorf("Storage conf missing nfs_shared_dir")
 	}
 	output, err := procutils.NewCommand(
 		"mount", "-t", "nfs", fmt.Sprintf("%s:%s", host, sharedDir), s.Path).RunWithTimeout(10 * time.Second)